Alex Galarce
5b88185611
[daip2-9] basic code refactor - unreferenced functions and reorganize existing files * remove unreferenced functions in ac_funcs.py * move compare_output.py * move file_counting.py * move merge_issue_regex.py * remove unreferenced functions in postprocessing_funcs.py * clean up preprocessing_funcs.py * move prerun.py * move table_analysis.py and textract_template.py to scripts * remove unreferenced functions in table_funcs.py * moved tin_pull.py to scripts * move detect_complex.py and remove merge_funcs.py * clean up dict operations and utils, mark functions in utils for destinations * removed errant import Approved-by: Katon Minhas
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
import table_funcs
|
|
import config
|
|
import utils
|
|
import preprocessing_funcs
|
|
|
|
import concurrent.futures
|
|
import pandas as pd
|
|
import os
|
|
|
|
input_dict = utils.read_input()
|
|
table_analysis_dicts = []
|
|
|
|
|
|
def table_analysis(file_object):
|
|
filename, contract_text = file_object
|
|
text_dict = preprocessing_funcs.split_text(contract_text)
|
|
file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")}
|
|
if utils.contains_reimbursement(contract_text):
|
|
file_dict["Contains Reimbursement"] = True
|
|
else:
|
|
file_dict["Contains Reimbursement"] = False
|
|
stats_dict = table_funcs.get_table_stats(text_dict)
|
|
file_dict.update(stats_dict)
|
|
table_analysis_dicts.append(file_dict)
|
|
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
|
futures = [executor.submit(table_analysis, item) for item in input_dict.items()]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
try:
|
|
future.result()
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
|
|
table_analysis_df = pd.DataFrame(table_analysis_dicts)
|
|
table_analysis_df.to_csv(
|
|
os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME)
|
|
)
|