doczyai-pipelines/fieldExtraction/src/utils/io_utils.py

import os
from io import StringIO

import pandas as pd
from botocore.exceptions import ClientError
from pyxlsb import open_workbook

import src.tracking.tracking_utils as tracking_utils
from src import config
from src.investment.code_funcs import crosswalk_levels, get_mappings
import src.constants.investment_values as investment_values
from src.utils.string_utils import datetime_str


def read_local(file_path): # io_utils.py
    # Check if the file is a text file
    if os.path.isfile(file_path) and file_path.endswith(".txt"):
        try:
            # First attempt to open the file with UTF-8 encoding
            with open(file_path, "r", encoding="utf-8") as file:
                file_contents = file.read()
                return file_contents
        except UnicodeDecodeError:
            # If UTF-8 fails, try reading the file with ANSI encoding
            try:
                with open(file_path, "r", encoding="cp1252") as file:
                    file_contents = file.read()
                    return file_contents
            except UnicodeDecodeError:
                # If ANSI also fails, log an error message or handle it accordingly
                print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.")

    if os.path.isfile(file_path) and file_path.endswith(".csv"):
        try:
            df = pd.read_csv(file_path)
            return df
        except Exception:
            print('Failed to read .csv file.')

    if os.path.isfile(file_path) and file_path.endswith(".xlsx"):
        try:
            df = pd.read_excel(file_path)
            return df
        except Exception:
            print('Failed to read .xlsx file.')


def list_s3_files(prefix=config.S3_PREFIX, bucket=config.S3_BUCKET): # io_utils.py
    """List all files in an S3 folder."""
    response = config.S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix)
    return [item["Key"] for item in response.get("Contents", [])]


def read_s3():
    s3_client = config.S3_CLIENT
    bucket = config.S3_BUCKET
    prefix = config.S3_PREFIX

    file_list = []
    continuation_token = None

    while True:

        if continuation_token:
            response = s3_client.list_objects_v2(
                Bucket=bucket, Prefix=prefix, ContinuationToken=continuation_token
            )
        else:
            response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)

        file_list += [
            obj["Key"]
            for obj in response.get("Contents", [])
            if not obj["Key"].endswith("/")
        ]

        if response.get("IsTruncated"):
            continuation_token = response.get("NextContinuationToken")
        else:
            break

    contract_list = sorted(file_list)
    files = {}

    for contract in contract_list:
        print(f"Reading {contract}")
        try:
            data = s3_client.get_object(Bucket=bucket, Key=contract)
            if contract.endswith('.txt'):
                contents = data["Body"].read()
                context = contents.decode("utf-8")
                path, filename = os.path.split(contract)
                files[filename] = context
        except:
            pass

    return files


def read_s3_csv(csv_path): # io_utils.py
    s3_client = config.S3_CLIENT
    s3_parts = csv_path[5:].split("/", 1)
    bucket = s3_parts[0]
    prefix = s3_parts[1] if len(s3_parts) > 1 else ""
    try:
        for obj in s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)["Contents"]:
            key = obj["Key"]
            if key.endswith(".csv"):
                response = s3_client.get_object(Bucket=bucket, Key=key)
                df = pd.read_csv(StringIO(response["Body"].read().decode("utf-8")))
                return df
    except ClientError as e:
        print(f"Error listing S3 objects: {e}")


def read_input(local_path=config.LOCAL_PATH): # io_utils.py
    if config.READ_MODE == "local":
        files = {}
        for file in os.listdir(local_path):
            full_path = os.path.join(local_path, file)
            file_text = read_local(full_path)
            files[file] = file_text
        return files
    elif config.READ_MODE == "s3":
        return read_s3()


def read_input_csv(): # io_utils.py
    if config.DF_READ_MODE == "local":
        ac_df = read_local(config.AC_DF) if config.AC_DF else None
        b_df = read_local(config.B_DF) if config.B_DF else None
    elif config.DF_READ_MODE == "s3":
        ac_df =  read_s3_csv(config.AC_DF) if config.AC_DF else None
        b_df = read_s3_csv(config.B_DF) if config.B_DF else None

    return ac_df, b_df


def filter_already_processed(input_dict, input_dict_b): # io_utils.py
    """Filter out already processed files based on either local files or S3 master tracking"""
    if config.WRITE_TO_S3:
        # Get processed files from S3 master tracking for this specific batch
        already_processed_ac, already_processed_b = tracking_utils.get_processed_files(config.BATCH_ID)

        # Keep files that still need either AC or B processing
        input_dict_ac = {
            k: v for k, v in input_dict.items()
            if k not in already_processed_ac
        }

        input_dict_b = {
            k: v for k, v in input_dict_b.items()
            if k not in already_processed_b
        }

        print(f"Filtering based on S3 master tracking for batch {config.BATCH_ID}:")
        print(f"AC: Found {len(already_processed_ac)} already processed, {len(input_dict_ac)} remaining")
        print(f"B: Found {len(already_processed_b)} already processed, {len(input_dict_b)} remaining")

    else:
        # Original local directory checking logic
        already_processed_ac, already_processed_b = [], []
        for folder_name in os.listdir(config.OUTPUT_DIRECTORY):
            folder_path = os.path.join(config.OUTPUT_DIRECTORY, folder_name)
            if os.path.isdir(folder_path):
                if config.AC_RESULTS_NAME in os.listdir(folder_path):
                    already_processed_ac.append(folder_name + ".txt")
                if config.B_RESULTS_NAME in os.listdir(folder_path):
                    already_processed_b.append(folder_name + ".txt")

        input_dict_ac = {
            k: v for k, v in input_dict.items()
            if k not in already_processed_ac
        }

        input_dict_b = {
            k: v for k, v in input_dict_b.items()
            if k not in already_processed_b
        }

        print(f"Filtering based on local output directory:")
        print(f"AC: Found {len(already_processed_ac)} already processed, {len(input_dict_ac)} remaining")
        print(f"B: Found {len(already_processed_b)} already processed, {len(input_dict_b)} remaining")

    return input_dict_ac, input_dict_b


def remove_txt_extension(filename): # io_utils.py
    if isinstance(filename, str) and filename.lower().endswith(".txt"):
        return filename[:-4]
    return filename


def read_xlsb(path): # io_utils.py
    with open_workbook(path) as wb:
        with wb.get_sheet(1) as sheet:
            rows = []
            for row in sheet.rows():
                rows.append([item.v for item in row])  # Extracting cell values
            df = pd.DataFrame(rows[1:], columns=rows[0])  # Converting to DataFrame, assuming first row is header
    return df

def write_local(df, filename, run_timestamp, output_type):
    """
    Saves a DataFrame as a CSV file in a local directory.

    Args:
        df (pd.DataFrame): The DataFrame to save.
        run_timestamp (str): The timestamp used to organize output files.

    The function creates a directory based on the configured output path 
    and the provided timestamp, then saves the DataFrame as a CSV file 
    named "<BATCH_ID>-RESULTS.csv" within it.
    """
    if output_type == "final":
        output_dir = os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, run_timestamp)
        os.makedirs(output_dir, exist_ok=True)
        df.to_csv(os.path.join(output_dir, f'{config.BATCH_ID}-RESULTS.csv'))
    elif output_type == "individual":
        base_filename = os.path.splitext(filename)[0].strip()
        os.makedirs(config.OUTPUT_DIRECTORY, exist_ok=True)
        df.to_csv(os.path.join(config.OUTPUT_DIRECTORY, f'{base_filename}_-RESULTS.csv'), index=False)


def write_s3(df, filename, run_timestamp, output_type):
    """
    Uploads a DataFrame to an S3 bucket as a CSV file.

    Args:
        df (pd.DataFrame): The DataFrame to be written to S3.
        run_timestamp (str): A timestamp representing the current run.

    The function generates an S3 key based on the configured BATCH_ID and 
    uploads the DataFrame as a CSV file to the configured S3 bucket. 
    The CSV file is saved in the "consolidated" folder with the naming 
    convention "<BATCH_ID>-RESULTS.csv".

    Configurations used:
        - config.BATCH_ID: Identifier for the batch process.
        - config.S3_CLIENT: The AWS S3 client for performing operations.
        - config.S3_OUTPUT_BUCKET: The target S3 bucket for storing the file.

    Prints:
        Confirmation message with the S3 key of the uploaded file.
    """
    if output_type == "final":
        output_path = f"{config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-RESULTS.csv"
    elif output_type == "individual":
        output_path = f"{config.BATCH_ID}/{run_timestamp}/individual/{filename}-RESULTS.csv"
    elif output_type == "error":
        output_path = f"{config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-ERRORS.csv"
    else:
        # Fallback path for unknown output types
        output_path = f"{config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-unknown-{output_type}.csv"

    csv_buffer = df.to_csv(index=False).encode()
    try:
        config.S3_CLIENT.put_object(
            Bucket=config.S3_OUTPUT_BUCKET,
            Key=output_path,
            Body=csv_buffer
        )
    except ClientError as e:
        print(f"Error uploading to S3: {e}")

    if output_type == "final":
        print(f"Saved batch output file: {output_path}")
    elif output_type == "individual":
        print(f"Saved individual output file: {output_path}")


def load_all_dataset():
    """_summary_: loads all csv files required for processing contracts including crosswalk files
    and mappings (embeddings, past results may be added later)
    """
    all_dataset = {}
    all_dataset['proc_codes_crosswalk'] = crosswalk_levels("proc_cd", investment_values.proc_levels)
    all_dataset['diag_codes_crosswalk'] = crosswalk_levels("diag_cd", investment_values.diag_levels)
    all_dataset['codes_mappings'] = get_mappings()

    return all_dataset

def load_embeddings():
    """loads all embedding files required for processing indirect codes (e.g. proc codes)
    """

    # list all embeddings present in s3
    s3_client = config.S3_CLIENT
    response = s3_client.list_objects_v2(Bucket="doczy-investment", Prefix="embeddings")
    file_list = [
        obj["Key"]
        for obj in response.get("Contents", [])
        if not obj["Key"].endswith("/")
    ]

    # list embeddings already present locally
    if not os.path.exists("embeddings"):
        os.makedirs("embeddings")
    local_files = []
    for root, dirs, files in os.walk("embeddings"):
        for name in files:
            local_files.append(os.path.join(root, name))

    # download embeddings not present locally
    for file in file_list:
        local_path = file

        # create directory structure for this file
        directory = os.path.dirname(local_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        
        # check if file already exists
        if not os.path.exists(local_path):
            try:
                print(f"{datetime_str()} Downloading {file} to {local_path}...")
                s3_client.download_file("doczy-investment", file, local_path)
            except Exception as e:
                print(f"{datetime_str()} Error downloading {file}: {e}")