jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

mvsrc

Browse Source
This commit is contained in:
Michael McGuinness
2024-09-27 14:45:35 +01:00
parent 27308f7d1b
commit a11fe060e6
30 changed files with 195 additions and 2948 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/file_processing.py
+9 -33
View File
@@ -4,8 +4,9 @@ import pandas as pd
import config
import preprocess
import table_funcs
import bottom_up_funcs
import top_down_funcs
import bottom_up_funcs
import postprocess_funcs
import postprocess
import merge_funcs
@@ -40,11 +41,6 @@ def process_file(file_object):
if config.VERBOSE:
print(f"Processing {filename}...")
################## CREATE OUTPUT DIRECTORIES ##################
base_filename = os.path.splitext(filename)[0].strip()
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
os.makedirs(output_dir, exist_ok=True)
################## PREPROCESS ##################
contract_text = preprocess.clean_newlines(contract_text)
contract_text = preprocess.clean_billed_charges(contract_text)
@@ -53,40 +49,20 @@ def process_file(file_object):
text_dict = preprocess.highlight_rates(text_dict)
################## RUN TOP DOWN ##################
td_results = top_down_funcs.run_top_down(
filename, text_dict
) # Returns list of dictionaries for each page
pd.DataFrame(td_results).to_csv(
os.path.join(output_dir, config.TD_RESULTS_NAME), index=False
)
td_results = top_down_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page
print(f"Top Down Complete - {filename}")
################## RUN BOTTOM UP ##################
bu_results = bottom_up_funcs.run_bottom_up(
filename, text_dict
) # Returns list of dictionaries
pd.DataFrame(bu_results).to_csv(
os.path.join(output_dir, config.BU_RESULTS_NAME), index=False
)
bu_results = bottom_up_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries
print(f"Bottom Up Complete - {filename}")
################## COMBINE TOP DOWN AND BOTTOM UP ##################
combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict)
print(f"TD/BU Merge Complete - {filename} ")
################## WRITE UNPROCESSED OUTPUT ##################
combined_df = pd.DataFrame(combined_results)
combined_df.to_csv(
os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False
)
################## RUN POSTPROCESSING ##################
# combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated
combined_df = pd.DataFrame(combined_results)
post_processed_combined_df = postprocess.postprocess_results(combined_df)
post_processed_combined_df.to_csv(
os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False
)
print(f"Postprocessing Complete - {filename} ")
print(f"Output Complete - {filename} ")
return post_processed_combined_df