39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
|
import table_funcs
|
||
|
|
import config
|
||
|
|
import utils
|
||
|
|
import preprocessing_funcs
|
||
|
|
|
||
|
|
import concurrent.futures
|
||
|
|
import pandas as pd
|
||
|
|
import os
|
||
|
|
|
||
|
|
input_dict = utils.read_input()
|
||
|
|
table_analysis_dicts = []
|
||
|
|
|
||
|
|
|
||
|
|
def table_analysis(file_object):
|
||
|
|
filename, contract_text = file_object
|
||
|
|
text_dict = preprocessing_funcs.split_text(contract_text)
|
||
|
|
file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")}
|
||
|
|
if utils.contains_reimbursement(contract_text):
|
||
|
|
file_dict["Contains Reimbursement"] = True
|
||
|
|
else:
|
||
|
|
file_dict["Contains Reimbursement"] = False
|
||
|
|
stats_dict = table_funcs.get_table_stats(text_dict)
|
||
|
|
file_dict.update(stats_dict)
|
||
|
|
table_analysis_dicts.append(file_dict)
|
||
|
|
|
||
|
|
|
||
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||
|
|
futures = [executor.submit(table_analysis, item) for item in input_dict.items()]
|
||
|
|
for future in concurrent.futures.as_completed(futures):
|
||
|
|
try:
|
||
|
|
future.result()
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error: {e}")
|
||
|
|
|
||
|
|
table_analysis_df = pd.DataFrame(table_analysis_dicts)
|
||
|
|
table_analysis_df.to_csv(
|
||
|
|
os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME)
|
||
|
|
)
|