Michael McGuinness
93 lines
3.5 KiB
Python
93 lines
3.5 KiB
Python
import os
|
|
import pandas as pd
|
|
|
|
import config
|
|
import preprocess
|
|
import table_funcs
|
|
import bottom_up_funcs
|
|
import top_down_funcs
|
|
import postprocess
|
|
import merge_funcs
|
|
|
|
|
|
def process_file(file_object):
|
|
"""
|
|
file_object = (filename , "file contents in one string"}
|
|
|
|
Processes a single file object containing text data from a contract.
|
|
|
|
This function orchestrates the full processing workflow for a single document, including:
|
|
- Directory setup for outputs based on the filename.
|
|
- Preprocessing text to clean and split it into manageable parts.
|
|
- Running 'Top Down' and 'Bottom Up' analysis using pre-defined models.
|
|
- Merging and consolidating results from both analysis methods.
|
|
- Postprocessing the consolidated results for final output.
|
|
|
|
Parameters:
|
|
file_object (tuple): A tuple containing the filename and the text content of the file.
|
|
|
|
Outputs:
|
|
- Several CSV files are generated in the designated output directory:
|
|
1. Results from the Top Down analysis.
|
|
2. Results from the Bottom Up analysis.
|
|
3. Combined results of both analyses before and after postprocessing.
|
|
|
|
Each step of the process is logged verbosely if the configuration is set to verbose mode.
|
|
"""
|
|
|
|
################## INITIATE PROCESSING ##################
|
|
filename, contract_text = file_object
|
|
if config.VERBOSE:
|
|
print(f"Processing {filename}...")
|
|
|
|
################## CREATE OUTPUT DIRECTORIES ##################
|
|
base_filename = os.path.splitext(filename)[0].strip()
|
|
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
################## PREPROCESS ##################
|
|
contract_text = preprocess.clean_newlines(contract_text)
|
|
contract_text = preprocess.clean_billed_charges(contract_text)
|
|
text_dict = preprocess.split_text(contract_text)
|
|
text_dict = table_funcs.align_and_format_tables(text_dict)
|
|
text_dict = preprocess.highlight_rates(text_dict)
|
|
|
|
################## RUN TOP DOWN ##################
|
|
td_results = top_down_funcs.run_top_down(
|
|
filename, text_dict
|
|
) # Returns list of dictionaries for each page
|
|
pd.DataFrame(td_results).to_csv(
|
|
os.path.join(output_dir, config.TD_RESULTS_NAME), index=False
|
|
)
|
|
print(f"Top Down Complete - {filename}")
|
|
|
|
################## RUN BOTTOM UP ##################
|
|
bu_results = bottom_up_funcs.run_bottom_up(
|
|
filename, text_dict
|
|
) # Returns list of dictionaries
|
|
pd.DataFrame(bu_results).to_csv(
|
|
os.path.join(output_dir, config.BU_RESULTS_NAME), index=False
|
|
)
|
|
print(f"Bottom Up Complete - {filename}")
|
|
|
|
################## COMBINE TOP DOWN AND BOTTOM UP ##################
|
|
combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict)
|
|
print(f"TD/BU Merge Complete - {filename} ")
|
|
|
|
################## WRITE UNPROCESSED OUTPUT ##################
|
|
combined_df = pd.DataFrame(combined_results)
|
|
combined_df.to_csv(
|
|
os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False
|
|
)
|
|
|
|
################## RUN POSTPROCESSING ##################
|
|
# combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated
|
|
|
|
post_processed_combined_df = postprocess.postprocess_results(combined_df)
|
|
post_processed_combined_df.to_csv(
|
|
os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False
|
|
)
|
|
print(f"Postprocessing Complete - {filename} ")
|
|
|
|
print(f"Output Complete - {filename} ")
|