Alex Galarce
5b88185611
[daip2-9] basic code refactor - unreferenced functions and reorganize existing files * remove unreferenced functions in ac_funcs.py * move compare_output.py * move file_counting.py * move merge_issue_regex.py * remove unreferenced functions in postprocessing_funcs.py * clean up preprocessing_funcs.py * move prerun.py * move table_analysis.py and textract_template.py to scripts * remove unreferenced functions in table_funcs.py * moved tin_pull.py to scripts * move detect_complex.py and remove merge_funcs.py * clean up dict operations and utils, mark functions in utils for destinations * removed errant import Approved-by: Katon Minhas
32 lines
926 B
Python
32 lines
926 B
Python
import utils
|
|
import config
|
|
|
|
import pandas as pd
|
|
|
|
print(f"Input Dir: {config.LOCAL_PATH}")
|
|
print(f"S3 Bucket: {config.S3_BUCKET}")
|
|
print(f"S3 Prefix: {config.S3_PREFIX}")
|
|
|
|
input_dict = (
|
|
utils.read_input()
|
|
) # keys are contract names, values are full contract text
|
|
total_files = len(input_dict)
|
|
|
|
print(f"Total Input Files : {len(input_dict)}")
|
|
|
|
# Filter B
|
|
input_dict_b = {
|
|
k: v for k, v in input_dict.items() if utils.contains_reimbursement(str(v))
|
|
} # Filter out non-contracts
|
|
print(
|
|
f"B Input Files after reimbursement filter: {len(input_dict_b)} | {total_files-len(input_dict_b)} files removed"
|
|
)
|
|
|
|
# Filter already processed
|
|
if config.FILTER_ALREADY_PROCESSED:
|
|
input_dict_ac, input_dict_b = utils.filter_already_processed(
|
|
input_dict, input_dict_b
|
|
)
|
|
print(f"AC Input Files left to be processed : {len(input_dict_ac)}")
|
|
print(f"B Input Files left to be processed : {len(input_dict_b)}")
|