Alex Galarce
5b88185611
[daip2-9] basic code refactor - unreferenced functions and reorganize existing files * remove unreferenced functions in ac_funcs.py * move compare_output.py * move file_counting.py * move merge_issue_regex.py * remove unreferenced functions in postprocessing_funcs.py * clean up preprocessing_funcs.py * move prerun.py * move table_analysis.py and textract_template.py to scripts * remove unreferenced functions in table_funcs.py * moved tin_pull.py to scripts * move detect_complex.py and remove merge_funcs.py * clean up dict operations and utils, mark functions in utils for destinations * removed errant import Approved-by: Katon Minhas
147 lines
4.8 KiB
Python
147 lines
4.8 KiB
Python
import os
|
|
import re
|
|
import csv
|
|
|
|
|
|
def split_text(text):
|
|
text_list = text.split("Start of Page No. = ")
|
|
text_dict = {page.split()[0]: page for page in text_list[1:]}
|
|
return text_dict
|
|
|
|
|
|
def contains_reimbursement(text, page):
|
|
if isinstance(text, dict):
|
|
return page.isdigit() and (
|
|
"%" in text[page] or "$" in text[page] or "percent " in text[page].lower()
|
|
)
|
|
elif isinstance(text, str):
|
|
return "%" in text or "$" in text or "percent " in text.lower()
|
|
else:
|
|
print("contains_reimbursement - Invalid data type")
|
|
return False
|
|
|
|
|
|
def search_files(directory):
|
|
pattern = re.compile(r"'',\s*'',")
|
|
results = []
|
|
total_files = 0
|
|
files_with_pattern = 0
|
|
|
|
for root, dirs, files in os.walk(directory):
|
|
for file in files:
|
|
if file.endswith(".txt"):
|
|
total_files += 1
|
|
file_path = os.path.join(root, file)
|
|
try:
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
pages = split_text(content)
|
|
|
|
reimbursement_pages = [
|
|
page
|
|
for page in pages
|
|
if contains_reimbursement(pages, page)
|
|
]
|
|
table_pages = [
|
|
page
|
|
for page in reimbursement_pages
|
|
if "-------Table Start--------" in pages[page]
|
|
]
|
|
|
|
matches = []
|
|
for page in table_pages:
|
|
matches.extend(pattern.findall(pages[page]))
|
|
|
|
if matches:
|
|
files_with_pattern += 1
|
|
results.append(
|
|
(
|
|
file_path,
|
|
len(matches),
|
|
True,
|
|
len(reimbursement_pages),
|
|
len(table_pages),
|
|
)
|
|
)
|
|
else:
|
|
results.append(
|
|
(
|
|
file_path,
|
|
0,
|
|
False,
|
|
len(reimbursement_pages),
|
|
len(table_pages),
|
|
)
|
|
)
|
|
except Exception as e:
|
|
print(f"Error reading {file_path}: {str(e)}")
|
|
results.append((file_path, 0, False, 0, 0))
|
|
|
|
return results, total_files, files_with_pattern
|
|
|
|
|
|
def write_to_csv(results, total_files, files_with_pattern, output_file):
|
|
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
|
|
writer = csv.writer(csvfile)
|
|
writer.writerow(
|
|
[
|
|
"File Name",
|
|
"Match Count",
|
|
"Contains Pattern",
|
|
"Reimbursement Pages",
|
|
"Table Pages",
|
|
"Percentage of Total",
|
|
]
|
|
)
|
|
|
|
for (
|
|
file_path,
|
|
match_count,
|
|
contains_pattern,
|
|
reimbursement_pages,
|
|
table_pages,
|
|
) in results:
|
|
percentage = (match_count / total_files) * 100 if total_files > 0 else 0
|
|
writer.writerow(
|
|
[
|
|
file_path,
|
|
match_count,
|
|
contains_pattern,
|
|
reimbursement_pages,
|
|
table_pages,
|
|
f"{percentage:.2f}%",
|
|
]
|
|
)
|
|
|
|
percentage_with_pattern = (
|
|
(files_with_pattern / total_files) * 100 if total_files > 0 else 0
|
|
)
|
|
writer.writerow(["", "", "", "", "", ""])
|
|
writer.writerow(["Summary", "", "", "", "", ""])
|
|
writer.writerow(["Total Files", total_files, "", "", "", ""])
|
|
writer.writerow(
|
|
[
|
|
"Files with Pattern",
|
|
files_with_pattern,
|
|
"",
|
|
"",
|
|
"",
|
|
f"{percentage_with_pattern:.2f}%",
|
|
]
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
facility_dir = "batch1_priority"
|
|
|
|
if os.path.isdir(facility_dir):
|
|
results, total_files, files_with_pattern = search_files(facility_dir)
|
|
output_file = os.path.join(current_dir, "contract_analysis_results.csv")
|
|
write_to_csv(results, total_files, files_with_pattern, output_file)
|
|
print(f"Results have been written to {output_file}")
|
|
else:
|
|
print(f"Error: The directory {facility_dir} does not exist.")
|