doczyai-pipelines/fieldExtraction/scripts/file_counting.py

import utils
import config

import pandas as pd

print(f"Input Dir: {config.LOCAL_PATH}")
print(f"S3 Bucket: {config.S3_BUCKET}")
print(f"S3 Prefix: {config.S3_PREFIX}")

input_dict = (
    utils.read_input()
)  # keys are contract names, values are full contract text
total_files = len(input_dict)

print(f"Total Input Files : {len(input_dict)}")

# Filter B
input_dict_b = {
    k: v for k, v in input_dict.items() if utils.contains_reimbursement(str(v))
}  # Filter out non-contracts
print(
    f"B Input Files after reimbursement filter: {len(input_dict_b)} | {total_files-len(input_dict_b)} files removed"
)

# Filter already processed
if config.FILTER_ALREADY_PROCESSED:
    input_dict_ac, input_dict_b = utils.filter_already_processed(
        input_dict, input_dict_b
    )
    print(f"AC Input Files left to be processed : {len(input_dict_ac)}")
    print(f"B Input Files left to be processed : {len(input_dict_b)}")