Merged in feature/doczy_reports (pull request #911)
Feature/doczy reports * Added post_doczy_reports * black format fix * Merged DEV into feature/doczy_reports * fixed code review Approved-by: Katon Minhas
This commit is contained in:
Rahul Ailaboina
committed by
Katon Minhas
Katon Minhas
parent
ddd1703e2f
commit
c9cd7ddb43
@@ -404,6 +404,25 @@ DTC_OUTPUT_FILE = (
|
||||
)
|
||||
DTC_JSON_OUTPUT_FOLDER = "dtc_json_results" # Folder for individual JSON results
|
||||
|
||||
############## PRE-DOCZY REPORT SETTINGS ##############
|
||||
PERFORM_PRE_DOCZY = (
|
||||
get_arg_value("perform_pre_doczy", "False") == "True"
|
||||
) # Enable Pre-Doczy classification + keyword report
|
||||
|
||||
############## POST-DOCZY VALIDATION REPORT SETTINGS ##############
|
||||
PERFORM_POST_DOCZY = (
|
||||
get_arg_value("perform_post_doczy", "False") == "True"
|
||||
) # Enable post-Doczy validation report
|
||||
PERFORM_POST_DOCZY_EXCEL = (
|
||||
get_arg_value("perform_post_doczy_excel", "False") == "True"
|
||||
) # Enable Excel-only post-Doczy report (no raw text files needed)
|
||||
POST_DOCZY_JSON_OUTPUT_FOLDER = (
|
||||
"post_doczy_json_results" # Folder for individual JSON results
|
||||
)
|
||||
DOCZY_RESULTS_PATH = get_arg_value(
|
||||
"doczy_results_path", ""
|
||||
) # Path to Doczy extraction output CSV/Excel (local path or S3 URI)
|
||||
|
||||
############## DASHBOARD POSTPROCESSING ##############
|
||||
# Run dashboard postprocessing (comma-separated list format). Default: CC only.
|
||||
# Set to True via run_dashboard arg when dashboard output is needed.
|
||||
|
||||
@@ -109,6 +109,23 @@ EFFECTIVE_DATE_FORMAT_PATTERNS = [
|
||||
r"\b(?:19|20)\d{2}[/\-](?:0?[1-9]|1[0-2])[/\-](?:0?[1-9]|[12]\d|3[01])\b",
|
||||
]
|
||||
|
||||
# ── Signature Patterns (for Post-Doczy Report) ──
|
||||
# Keyword patterns that indicate signature presence in contract documents.
|
||||
SIGNATURE_KEYWORD_PATTERNS = [
|
||||
r"\bsignature\b",
|
||||
r"\bsigned\s+by\b",
|
||||
r"\bauthorized\s+signature\b",
|
||||
r"\bexecuted\s+by\b",
|
||||
r"\bsignatory\b",
|
||||
r"\bIN\s+WITNESS\s+WHEREOF\b",
|
||||
r"\bsigning\s+authority\b",
|
||||
r"\bduly\s+authorized\b",
|
||||
r"\bby\s*:\s*_+",
|
||||
r"\bsign\s+here\b",
|
||||
r"\bsigned\s+and\s+sealed\b",
|
||||
r"\bexecuted\s+this\b",
|
||||
]
|
||||
|
||||
# dba patterns
|
||||
DBA_PATTERNS = [r"\bD/B/A\b", r"\bDBA\b", r"\bDOING BUSINESS AS\b", r"\bD B A\b"]
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+101
-5
@@ -4,7 +4,6 @@ import os
|
||||
import re
|
||||
import tempfile
|
||||
from io import StringIO
|
||||
import json
|
||||
from threading import Lock
|
||||
|
||||
import pandas as pd
|
||||
@@ -336,6 +335,47 @@ def read_s3_csv(csv_path) -> pd.DataFrame | None:
|
||||
return None
|
||||
|
||||
|
||||
def read_s3_excel(excel_path) -> pd.DataFrame | None:
|
||||
"""Reads an Excel (.xlsx) file from an S3 bucket and returns it as a DataFrame.
|
||||
|
||||
Downloads the file to a temporary location since pd.read_excel requires
|
||||
a file-like object or path, not a raw byte stream with S3 chunking.
|
||||
|
||||
Args:
|
||||
excel_path (str): The S3 URL of the Excel file
|
||||
(e.g., s3://bucket-name/path/to/file.xlsx).
|
||||
|
||||
Returns:
|
||||
pd.DataFrame | None: The contents of the Excel file as a pandas DataFrame,
|
||||
or None if an error occurs.
|
||||
"""
|
||||
s3_client = config.S3_CLIENT
|
||||
|
||||
if not excel_path.startswith("s3://"):
|
||||
logging.error(f"Invalid S3 path: {excel_path}")
|
||||
return None
|
||||
|
||||
s3_parts = excel_path[5:].split("/", 1)
|
||||
bucket = s3_parts[0]
|
||||
key = s3_parts[1] if len(s3_parts) > 1 else ""
|
||||
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=True) as tmp:
|
||||
s3_client.download_file(bucket, key, tmp.name)
|
||||
df = pd.read_excel(tmp.name)
|
||||
logging.info(
|
||||
f"Read Excel from S3: {excel_path} "
|
||||
f"({len(df)} rows, {len(df.columns)} columns)"
|
||||
)
|
||||
return df
|
||||
except ClientError as e:
|
||||
logging.error(f"Error downloading Excel from S3: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logging.error(f"Error reading Excel file from {excel_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def read_input(local_path=config.LOCAL_PATH) -> dict:
|
||||
"""Reads input files from the specified local path or S3 bucket.
|
||||
|
||||
@@ -796,6 +836,33 @@ def write_local(
|
||||
quoting=1,
|
||||
)
|
||||
return None
|
||||
elif output_type == "pre_doczy":
|
||||
output_dir = os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, run_timestamp)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
df.to_csv(
|
||||
os.path.join(output_dir, f"{config.BATCH_ID}-PRE-DOCZY.csv"),
|
||||
index=False,
|
||||
quoting=1,
|
||||
)
|
||||
return None
|
||||
elif output_type == "post_doczy":
|
||||
output_dir = os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, run_timestamp)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
df.to_csv(
|
||||
os.path.join(output_dir, f"{config.BATCH_ID}-POST-DOCZY.csv"),
|
||||
index=False,
|
||||
quoting=1,
|
||||
)
|
||||
return None
|
||||
elif output_type == "post_doczy_summary":
|
||||
output_dir = os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, run_timestamp)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
df.to_csv(
|
||||
os.path.join(output_dir, f"{config.BATCH_ID}-POST-DOCZY-SUMMARY.csv"),
|
||||
index=False,
|
||||
quoting=1,
|
||||
)
|
||||
return None
|
||||
elif output_type == "qc_qa_validated":
|
||||
# QC/QA validated output goes to outputs/qc_qa folder
|
||||
output_dir = os.path.join(config.QC_QA_OUTPUT_DIRECTORY, run_timestamp)
|
||||
@@ -842,7 +909,7 @@ def write_local(
|
||||
return None
|
||||
else:
|
||||
logging.error(
|
||||
f"Unknown output_type: {output_type}. Valid types are: 'final', 'individual', 'error', 'usage', 'usage_summary', 'dtc', 'dtc_summary', 'qc_qa_validated', 'qc_qa_stats', 'cc_results_full', 'dashboard_results_full', 'qc_qa_cc_full', 'qc_qa_error', 'parent_child', 'individual_cc'"
|
||||
f"Unknown output_type: {output_type}. Valid types are: 'final', 'individual', 'error', 'usage', 'usage_summary', 'dtc', 'dtc_summary', 'pre_doczy', 'post_doczy', 'post_doczy_summary', 'qc_qa_validated', 'qc_qa_stats', 'cc_results_full', 'dashboard_results_full', 'qc_qa_cc_full', 'qc_qa_error', 'parent_child', 'individual_cc'"
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -960,6 +1027,28 @@ def write_s3(
|
||||
config.S3_CLIENT.put_object(
|
||||
Bucket=config.S3_OUTPUT_BUCKET, Key=output_path, Body=csv_buffer
|
||||
)
|
||||
elif output_type == "pre_doczy":
|
||||
output_path = (
|
||||
f"{config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-PRE-DOCZY.csv"
|
||||
)
|
||||
csv_buffer = df.to_csv(index=False).encode()
|
||||
config.S3_CLIENT.put_object(
|
||||
Bucket=config.S3_OUTPUT_BUCKET, Key=output_path, Body=csv_buffer
|
||||
)
|
||||
elif output_type == "post_doczy":
|
||||
output_path = (
|
||||
f"{config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-POST-DOCZY.csv"
|
||||
)
|
||||
csv_buffer = df.to_csv(index=False).encode()
|
||||
config.S3_CLIENT.put_object(
|
||||
Bucket=config.S3_OUTPUT_BUCKET, Key=output_path, Body=csv_buffer
|
||||
)
|
||||
elif output_type == "post_doczy_summary":
|
||||
output_path = f"{config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-POST-DOCZY-SUMMARY.csv"
|
||||
csv_buffer = df.to_csv(index=False).encode()
|
||||
config.S3_CLIENT.put_object(
|
||||
Bucket=config.S3_OUTPUT_BUCKET, Key=output_path, Body=csv_buffer
|
||||
)
|
||||
elif output_type == "qc_qa_validated":
|
||||
output_path = f"{config.BATCH_ID}/{run_timestamp}/automation_qa-qc/{config.BATCH_ID}-QC-QA-VALIDATED.csv"
|
||||
csv_buffer = df.to_csv(index=False).encode()
|
||||
@@ -1039,6 +1128,12 @@ def write_s3(
|
||||
logging.info(f"Saved DTC output file: {output_path}")
|
||||
elif output_type == "dtc_summary":
|
||||
logging.info(f"Saved DTC summary file: {output_path}")
|
||||
elif output_type == "pre_doczy":
|
||||
logging.info(f"Saved Pre-Doczy report file: {output_path}")
|
||||
elif output_type == "post_doczy":
|
||||
logging.info(f"Saved Post-Doczy report file: {output_path}")
|
||||
elif output_type == "post_doczy_summary":
|
||||
logging.info(f"Saved Post-Doczy summary file: {output_path}")
|
||||
elif output_type == "qc_qa_validated":
|
||||
logging.info(f"Saved QC/QA validated data file: {output_path}")
|
||||
elif output_type == "qc_qa_stats":
|
||||
@@ -1157,14 +1252,15 @@ def save_result_to_json(filename, result, json_folder):
|
||||
def read_DataFrame(file_path: str) -> pd.DataFrame:
|
||||
"""
|
||||
Takes in a file path and returns a pandas DataFrame.
|
||||
Supports reading from local file system and only CSV from S3.
|
||||
Supports reading from local file system, and CSV/Excel from S3.
|
||||
"""
|
||||
|
||||
is_s3 = file_path.startswith("s3://")
|
||||
|
||||
if is_s3:
|
||||
file = read_s3_csv(file_path)
|
||||
return file
|
||||
if file_path.lower().endswith((".xlsx", ".xls")):
|
||||
return read_s3_excel(file_path)
|
||||
return read_s3_csv(file_path)
|
||||
file = read_local(file_path)
|
||||
return file
|
||||
|
||||
|
||||
Reference in New Issue
Block a user