226 lines
6.9 KiB
Python
226 lines
6.9 KiB
Python
|
import os
|
||
|
|
import re
|
||
|
|
import pandas as pd
|
||
|
|
import shutil
|
||
|
|
import time
|
||
|
|
import sys
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
import config
|
||
|
|
|
||
|
|
|
||
|
|
def read_local(file_path):
|
||
|
|
# Check if the file is a text file
|
||
|
|
if os.path.isfile(file_path) and file_path.endswith(".txt"):
|
||
|
|
try:
|
||
|
|
# First attempt to open the file with UTF-8 encoding
|
||
|
|
with open(file_path, "r", encoding="utf-8") as file:
|
||
|
|
file_contents = file.read()
|
||
|
|
return file_contents
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
# If UTF-8 fails, try reading the file with ANSI encoding
|
||
|
|
try:
|
||
|
|
with open(file_path, "r", encoding="cp1252") as file:
|
||
|
|
file_contents = file.read()
|
||
|
|
return file_contents
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
# If ANSI also fails, log an error message or handle it accordingly
|
||
|
|
print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.")
|
||
|
|
|
||
|
|
|
||
|
|
def list_s3_files(prefix=config.S3_PREFIX, bucket=config.S3_BUCKET):
|
||
|
|
"""List all files in an S3 folder."""
|
||
|
|
response = config.S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix)
|
||
|
|
return [item["Key"] for item in response.get("Contents", [])]
|
||
|
|
|
||
|
|
|
||
|
|
def read_s3():
|
||
|
|
s3_client = config.S3_CLIENT
|
||
|
|
bucket = config.S3_BUCKET
|
||
|
|
prefix = config.S3_PREFIX
|
||
|
|
|
||
|
|
file_list = []
|
||
|
|
continuation_token = None
|
||
|
|
|
||
|
|
while True:
|
||
|
|
|
||
|
|
if continuation_token:
|
||
|
|
response = s3_client.list_objects_v2(
|
||
|
|
Bucket=bucket, Prefix=prefix, ContinuationToken=continuation_token
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
|
||
|
|
|
||
|
|
file_list += [
|
||
|
|
obj["Key"]
|
||
|
|
for obj in response.get("Contents", [])
|
||
|
|
if not obj["Key"].endswith("/")
|
||
|
|
]
|
||
|
|
|
||
|
|
if response.get("IsTruncated"):
|
||
|
|
continuation_token = response.get("NextContinuationToken")
|
||
|
|
else:
|
||
|
|
break
|
||
|
|
|
||
|
|
contract_list = sorted(file_list)
|
||
|
|
files = {}
|
||
|
|
|
||
|
|
for contract in contract_list:
|
||
|
|
print(f"Reading {contract}")
|
||
|
|
try:
|
||
|
|
data = s3_client.get_object(Bucket=bucket, Key=contract)
|
||
|
|
contents = data["Body"].read()
|
||
|
|
context = contents.decode("utf-8")
|
||
|
|
path, filename = os.path.split(contract)
|
||
|
|
files[filename] = context
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
return files
|
||
|
|
|
||
|
|
|
||
|
|
def read_input(local_path=config.LOCAL_PATH):
|
||
|
|
if config.READ_MODE == "local":
|
||
|
|
files = {}
|
||
|
|
for file in os.listdir(local_path):
|
||
|
|
full_path = os.path.join(local_path, file)
|
||
|
|
file_text = read_local(full_path)
|
||
|
|
files[file] = file_text
|
||
|
|
return files
|
||
|
|
elif config.READ_MODE == "s3":
|
||
|
|
return read_s3()
|
||
|
|
|
||
|
|
|
||
|
|
def consolidate_individual(input_folder="results", output_folder="output"):
|
||
|
|
dfs = []
|
||
|
|
for filename in os.listdir(input_folder):
|
||
|
|
if filename.endswith(".csv"):
|
||
|
|
filepath = os.path.join(input_folder, filename)
|
||
|
|
dfs.append(pd.read_csv(filepath))
|
||
|
|
|
||
|
|
# Remove temp folder
|
||
|
|
if input_folder == "temp" and os.path.exists(input_folder):
|
||
|
|
shutil.rmtree(input_folder)
|
||
|
|
|
||
|
|
consolidated_df = pd.concat(dfs, ignore_index=True)
|
||
|
|
|
||
|
|
# Write output
|
||
|
|
version = 1
|
||
|
|
existing_files = [
|
||
|
|
filename
|
||
|
|
for filename in os.listdir(output_folder)
|
||
|
|
if filename.startswith(f"consolidated_results_{config.TODAY}")
|
||
|
|
]
|
||
|
|
if existing_files:
|
||
|
|
versions = [
|
||
|
|
int(file.split("_v")[1].split(".")[0])
|
||
|
|
for file in existing_files
|
||
|
|
if "_v" in file
|
||
|
|
]
|
||
|
|
if versions:
|
||
|
|
version = max(versions) + 1
|
||
|
|
filename = f"consolidated_results_{config.TODAY}_v{version}.csv"
|
||
|
|
consolidated_df.to_csv(os.path.join(output_folder, filename))
|
||
|
|
|
||
|
|
|
||
|
|
def preprocess_text_file(file_path):
|
||
|
|
# Attempt to open the file with UTF-8 encoding first
|
||
|
|
try:
|
||
|
|
with open(file_path, "r", encoding="utf-8") as file:
|
||
|
|
text = file.read()
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
# If UTF-8 fails, try reading the file with ANSI (cp1252) encoding
|
||
|
|
try:
|
||
|
|
with open(file_path, "r", encoding="cp1252") as file:
|
||
|
|
text = file.read()
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
# If ANSI also fails, log an error message or handle it accordingly
|
||
|
|
print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.")
|
||
|
|
return []
|
||
|
|
|
||
|
|
# Split the text into pages based on a specific marker
|
||
|
|
pages = re.split(r"Start of Page No\. = \d+", text)
|
||
|
|
return pages
|
||
|
|
|
||
|
|
|
||
|
|
def format_td_check(td_dicts, dont_include_list):
|
||
|
|
final_str = ""
|
||
|
|
dict_count = 1
|
||
|
|
for td_dict in td_dicts:
|
||
|
|
final_str += str(dict_count) + ". "
|
||
|
|
for k in td_dict.keys():
|
||
|
|
if k not in dont_include_list:
|
||
|
|
final_str += k + ": " + td_dict[k] + ", "
|
||
|
|
final_str += "\n"
|
||
|
|
dict_count += 1
|
||
|
|
return final_str
|
||
|
|
|
||
|
|
|
||
|
|
def contains_reimbursement(text, page="1"):
|
||
|
|
if isinstance(text, dict):
|
||
|
|
return page.isdigit() and (
|
||
|
|
"%" in text[page]
|
||
|
|
or "$" in text[page]
|
||
|
|
or "percent " in text[page]
|
||
|
|
or "compensation schedule" in text[page].lower()
|
||
|
|
or "reimbursement schedule" in text[page].lower()
|
||
|
|
)
|
||
|
|
elif isinstance(text, str):
|
||
|
|
return (
|
||
|
|
"%" in text
|
||
|
|
or "$" in text
|
||
|
|
or "percent " in text
|
||
|
|
or "compensation schedule" in text.lower()
|
||
|
|
or "reimbursement schedule" in text.lower()
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
print("contains_reimbursement - Invalid data type")
|
||
|
|
|
||
|
|
|
||
|
|
def contains_keyword(text, keywords):
|
||
|
|
# FILL OUT HERE
|
||
|
|
return
|
||
|
|
|
||
|
|
|
||
|
|
def filter_already_processed(input_dict, fields=config.FIELDS):
|
||
|
|
already_processed = []
|
||
|
|
for folder_name in os.listdir(config.OUTPUT_DIRECTORY):
|
||
|
|
# ABC
|
||
|
|
if (
|
||
|
|
fields == "abc"
|
||
|
|
and config.AC_RESULTS_NAME
|
||
|
|
in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name))
|
||
|
|
and config.B_RESULTS_NAME
|
||
|
|
in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name))
|
||
|
|
):
|
||
|
|
already_processed.append(folder_name + ".txt")
|
||
|
|
# Fill AC Here
|
||
|
|
if (
|
||
|
|
config.AC_RESULTS_NAME
|
||
|
|
in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name))
|
||
|
|
and config.FIELDS == "ac"
|
||
|
|
):
|
||
|
|
already_processed.append(folder_name + ".txt")
|
||
|
|
# Fill B Here
|
||
|
|
if (
|
||
|
|
config.B_RESULTS_NAME
|
||
|
|
in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name))
|
||
|
|
and config.FIELDS == "b"
|
||
|
|
):
|
||
|
|
already_processed.append(folder_name + ".txt")
|
||
|
|
|
||
|
|
input_dict = {
|
||
|
|
key: input_dict[key]
|
||
|
|
for key in input_dict.keys()
|
||
|
|
if key not in already_processed
|
||
|
|
}
|
||
|
|
return input_dict
|
||
|
|
|
||
|
|
|
||
|
|
def is_empty(value):
|
||
|
|
if pd.isna(value):
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
empty_values = [None, "", "N/A", "NA", "null", "none", "NaN", np.nan, "nan"]
|
||
|
|
return value in empty_values
|