Alex Galarce
5b88185611
[daip2-9] basic code refactor - unreferenced functions and reorganize existing files * remove unreferenced functions in ac_funcs.py * move compare_output.py * move file_counting.py * move merge_issue_regex.py * remove unreferenced functions in postprocessing_funcs.py * clean up preprocessing_funcs.py * move prerun.py * move table_analysis.py and textract_template.py to scripts * remove unreferenced functions in table_funcs.py * moved tin_pull.py to scripts * move detect_complex.py and remove merge_funcs.py * clean up dict operations and utils, mark functions in utils for destinations * removed errant import Approved-by: Katon Minhas
102 lines
3.0 KiB
Python
102 lines
3.0 KiB
Python
import table_funcs
|
|
import config
|
|
import utils
|
|
import preprocessing_funcs
|
|
import concurrent.futures
|
|
import pandas as pd
|
|
import os
|
|
|
|
input_dict = utils.read_input()
|
|
table_analysis_dicts = []
|
|
|
|
|
|
def table_analysis(file_object):
|
|
filename, contract_text = file_object
|
|
text_dict = preprocessing_funcs.split_text(contract_text)
|
|
file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")}
|
|
|
|
contains_reimbursement = utils.contains_reimbursement(contract_text)
|
|
file_dict["Contains Reimbursement"] = contains_reimbursement
|
|
|
|
# Determine if the file should be present in the output
|
|
file_dict["Present in Output"] = contains_reimbursement
|
|
|
|
stats_dict = table_funcs.get_table_stats(text_dict)
|
|
file_dict.update(stats_dict)
|
|
table_analysis_dicts.append(file_dict)
|
|
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
|
futures = [executor.submit(table_analysis, item) for item in input_dict.items()]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
try:
|
|
future.result()
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
|
|
# Create the main table analysis DataFrame
|
|
table_analysis_df = pd.DataFrame(table_analysis_dicts)
|
|
table_analysis_df.to_csv(
|
|
os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME)
|
|
)
|
|
|
|
# Define the columns we want in our report
|
|
report_columns = [
|
|
"Filename",
|
|
"Contains Reimbursement",
|
|
"Present in Output",
|
|
"Table Count",
|
|
"Table Page Count",
|
|
"Rate Count",
|
|
"Num Rate Pages",
|
|
"Num >=10 Rates",
|
|
]
|
|
|
|
# Create the new report DataFrame with available columns
|
|
report_df = table_analysis_df[
|
|
[col for col in report_columns if col in table_analysis_df.columns]
|
|
]
|
|
|
|
|
|
# Function to safely create summary columns
|
|
def create_summary_column(df, source_col, target_col):
|
|
if source_col in df.columns:
|
|
df[target_col] = df[source_col].apply(
|
|
lambda x: ", ".join(map(str, eval(x))) if x != "[]" else "None"
|
|
)
|
|
else:
|
|
df[target_col] = "N/A"
|
|
|
|
|
|
# Add summary columns if the data is available
|
|
create_summary_column(report_df, "Table Pages", "Table Pages Summary")
|
|
create_summary_column(report_df, "Rate Pages", "Rate Pages Summary")
|
|
create_summary_column(report_df, ">=10 Rate Pages", ">=10 Rate Pages Summary")
|
|
|
|
# Define the final column order
|
|
final_columns = [
|
|
"Filename",
|
|
"Contains Reimbursement",
|
|
"Present in Output",
|
|
"Table Count",
|
|
"Table Page Count",
|
|
"Table Pages Summary",
|
|
"Rate Count",
|
|
"Num Rate Pages",
|
|
"Rate Pages Summary",
|
|
"Num >=10 Rates",
|
|
">=10 Rate Pages Summary",
|
|
]
|
|
|
|
# Reorder columns, including only those that exist
|
|
report_df = report_df[[col for col in final_columns if col in report_df.columns]]
|
|
|
|
# Save the new report to a CSV file
|
|
report_output_path = os.path.join(
|
|
config.REPORTING_OUTPUT_DIRECTORY, "enhanced_reimbursement_report.csv"
|
|
)
|
|
report_df.to_csv(report_output_path, index=False)
|
|
|
|
print(f"Enhanced reimbursement report saved to: {report_output_path}")
|
|
print("Columns in the report:", ", ".join(report_df.columns))
|