Karan Desai
1098be5cf5
grouped keywords and IRS regex chunking * abandon old branch and rebuild * removed print for field names, prompts * pushing as part of leftovers * IRS regex chunking * Merged in feature/chunk_term_clean (pull request #262) chunk_log_added * chunk_log_added * Merge remote-tracking branch 'remotes/origin/feature_ac_chunking_clean' into feature/chunk_term_clean Approved-by: Alex Galarce * added function for regex based IRS chunking * shifted regex_match_chunk function to utils * added helper functions for regex based chunking * replaced tin_regex function to ac_funcs * removed prompt for irs_others * added regex chunking for irs * removed irs group from smart chunking * testing functionality for regex based irs fields * updated return N/A condition * Merged main into feature_ac_chunking_clean * file_processing.py edited online with Bitbucket * added back conditional prompts * minor fixed for PR * remove install types * mering clean branch * with passed test casses * added flag for regex based tin execution Approved-by: Michael McGuinness
32 lines
926 B
Python
32 lines
926 B
Python
import utils
|
|
import config
|
|
|
|
import pandas as pd
|
|
|
|
print(f"Input Dir: {config.LOCAL_PATH}")
|
|
print(f"S3 Bucket: {config.S3_BUCKET}")
|
|
print(f"S3 Prefix: {config.S3_PREFIX}")
|
|
|
|
input_dict = (
|
|
utils.read_input()
|
|
) # keys are contract names, values are full contract text
|
|
total_files = len(input_dict)
|
|
|
|
print(f"Total Input Files : {len(input_dict)}")
|
|
|
|
# Filter B
|
|
input_dict_b = {
|
|
k: v for k, v in input_dict.items() if utils.contains_reimbursement(str(v))
|
|
} # Filter out non-contracts
|
|
print(
|
|
f"B Input Files after reimbursement filter: {len(input_dict_b)} | {total_files-len(input_dict_b)} files removed"
|
|
)
|
|
|
|
# Filter already processed
|
|
if config.FILTER_ALREADY_PROCESSED:
|
|
input_dict_ac, input_dict_b = utils.filter_already_processed(
|
|
input_dict, input_dict_b
|
|
)
|
|
print(f"AC Input Files left to be processed : {len(input_dict_ac)}")
|
|
print(f"B Input Files left to be processed : {len(input_dict_b)}")
|