From cc3c7a7ae2899d34084ab8ffb960e191a0063181 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 24 Oct 2025 19:14:13 +0000 Subject: [PATCH] Merged in refactor/add-doc-classification (pull request #745) Add document classification * Add document classification * Update state-specific classification * Merge branch 'main' into refactor/add-doc-classification * Merge branch 'main' into refactor/add-doc-classification * functional for null state * standardize output name Approved-by: Karan Desai --- .../doc_classification_funcs.py | 88 +++++++++++++++++++ .../src/document_classification/main.py | 86 ++++++++++++++++++ .../src/investment/preprocessing_funcs.py | 16 ++++ 3 files changed, 190 insertions(+) create mode 100644 fieldExtraction/src/document_classification/doc_classification_funcs.py create mode 100644 fieldExtraction/src/document_classification/main.py diff --git a/fieldExtraction/src/document_classification/doc_classification_funcs.py b/fieldExtraction/src/document_classification/doc_classification_funcs.py new file mode 100644 index 0000000..35c4e35 --- /dev/null +++ b/fieldExtraction/src/document_classification/doc_classification_funcs.py @@ -0,0 +1,88 @@ + + +def apply_rules_by_state(row, state): + """ + Check if final verdict should be set as false based on state-specific rules. + Returns True if document should be excluded (final_verdict = False). + """ + if state is None: + return False + + filename = row['Contract Name'].upper() # Convert to uppercase for case-insensitive matching + pages = row['total_pages'] + + # Define state-specific exclusion keywords + state_rules = { + 'WI': { + 'max_pages': 75, + 'keywords': [ + 'W9', 'COM - LTR', 'FORM - FORM', 'FORM - LIC', 'FORM - OWN', + 'FORM - DOCS', 'FORM - APP', 'FORM - CRED', 'FORM - PPA', + 'FORM - WPA', 'CRF', 'APPLICATION', 'CREDENTIALING', + 'CRED DOCUMENTS', 'CRED DOCUMENT', ' PIF ', ' COLL ', ' ROS ', ' HDO ' + ] + }, + 'MA': { + 'max_pages': None, + 'keywords': [ + 'CRF', 'FORM - CRED', 'CREDENTIALING', 'COM - LTR', + 'FORM - APP', 'APPLICATION' + ] + }, + 'MS': { + 'max_pages': None, + 'keywords': [ + 'FORM - OWN', ' PIF ', 'FORM - LIC', 'FORM - DOCS', 'FORM PPA', ' COLL ' + ], + 'special_rules': { + 'W9': 6 # W9 with exactly 6 pages + } + }, + 'SC': { + 'max_pages': None, + 'keywords': [ + 'FORM - FORM', 'FORM - OWN', 'FORM - LIC', 'COM - LTR' + ], + 'special_rules': { + 'W9': 6 # W9 with exactly 6 pages + } + } + } + + state = state.upper() + if state not in state_rules: + return False + + rules = state_rules[state] + + # Check if state has a max page limit + if rules['max_pages'] is not None: + # For states with page limits (like WI) + if pages > rules['max_pages']: + return True # Exclude if pages exceed limit + + # For documents within page limit, check for exclusion keywords + for keyword in rules['keywords']: + if keyword in filename: + return True + + # Check special rules + if 'special_rules' in rules: + for keyword, required_pages in rules['special_rules'].items(): + if keyword in filename and pages == required_pages: + return True + else: + # For states without page limits (like MA, MS, SC) + # Check keywords regardless of page count + for keyword in rules['keywords']: + if keyword in filename: + return True + + # Check special rules + if 'special_rules' in rules: + for keyword, required_pages in rules['special_rules'].items(): + if keyword in filename and pages == required_pages: + return True + + # If no exclusion rules matched, don't exclude + return False \ No newline at end of file diff --git a/fieldExtraction/src/document_classification/main.py b/fieldExtraction/src/document_classification/main.py new file mode 100644 index 0000000..d782fee --- /dev/null +++ b/fieldExtraction/src/document_classification/main.py @@ -0,0 +1,86 @@ +import pandas as pd +import re +from src.investment.preprocessing_funcs import clean_newlines, split_text, clean_law_symbols +from src.document_classification import doc_classification_funcs +from src.utils import io_utils, string_utils +import src.config as config + +OUTPUT_FILE = f'Document-Classification-{config.BATCH_ID}.xlsx' + +def main(state=None): + + yes_keywords = ['Agreement', 'Amendment', 'Amend', 'Agmt', 'Agree', 'Agrmnt', 'Standard Clause Letter', 'BH Letter', 'Appendix', 'AMD', 'Notice', 'Contract'] + no_keywords = ['void', 'do not use', 'Internal memo', 'Disclosure', 'Disc of Owner', 'W9', 'W-9', 'Credential', 'Coversheet', 'Cvrsht', 'Thumbs.db', 'Questionnaire',' Cert ', 'Certificate', 'email', 'Exclusion', 'Change Letter', 'Termination Letter', 'Form', 'Provider Letter', '.xls', '.lnk', 'Taxpayer', 'Regulatory Deeming', 'Administrative Amendment', 'Administrative Agreement', 'Instructions', 'Adobe Sign', 'Notice of Legal', 'Receipt ', + "Dear", "to whom it may concern"] + + yes_patterns = [(kw, re.compile(re.escape(kw), re.IGNORECASE)) for kw in yes_keywords] + no_patterns = [(kw, re.compile(re.escape(kw), re.IGNORECASE)) for kw in no_keywords] + + record = [] + + files = io_utils.read_input() + + for filename, contract_text in files.items(): + + contract_text = clean_newlines(contract_text) + contract_text = clean_law_symbols(contract_text) + text_dict = split_text(contract_text) + + # --- search in filename --- + yes_keywords_in_filename = [kw for kw, pattern in yes_patterns if pattern.search(filename)] + no_keywords_in_filename = [kw for kw, pattern in no_patterns if pattern.search(filename)] + + # --- first 5 pages --- + threshold = min(5, len(text_dict)) + first_five_pages = "\n".join(text_dict[str(page)] for page in range(1, threshold + 1)) + + # --- search in body (first 5 pages only) --- + yes_keywords_in_body = [kw for kw, pattern in yes_patterns if pattern.search(first_five_pages)] + no_keywords_in_body = [kw for kw, pattern in no_patterns if pattern.search(first_five_pages)] + + filename_verdict = len(yes_keywords_in_filename) > 0 + body_verdict = len(yes_keywords_in_body) > 0 + final_verdict = filename_verdict or body_verdict + + record.append(( + filename, + len(text_dict), + yes_keywords_in_filename, + no_keywords_in_filename, + yes_keywords_in_body, + no_keywords_in_body, + filename_verdict, + body_verdict, + final_verdict, + )) + + + columns = [ + 'Contract Name', + 'total_pages', + 'yes_keywords_in_filename', + 'no_keywords_in_filename', + 'yes_keywords_in_body', + 'no_keywords_in_body', + 'filename_verdict', + 'body_verdict', + 'final_verdict' + ] + + df = pd.DataFrame(record, columns=columns) + + # final verdict + df['final_verdict'] = df['filename_verdict'] | df['body_verdict'] + df.loc[df['total_pages']<=3,'final_verdict'] = False + + # Apply state-specific filtering if state is provided + if not string_utils.is_empty(state): + exclusion_mask = df.apply(lambda row: doc_classification_funcs.apply_rules_by_state(row, state), axis=1) + df.loc[exclusion_mask, 'final_verdict'] = False + print(f"Applied state-specific filtering for {state}!") + + df.to_excel(OUTPUT_FILE, index=False) + + +if __name__ == "__main__": + main(state=config.STATE) \ No newline at end of file diff --git a/fieldExtraction/src/investment/preprocessing_funcs.py b/fieldExtraction/src/investment/preprocessing_funcs.py index 4ed88ba..afb4f7f 100644 --- a/fieldExtraction/src/investment/preprocessing_funcs.py +++ b/fieldExtraction/src/investment/preprocessing_funcs.py @@ -63,6 +63,22 @@ def clean_law_symbols(contract_text): return contract_text +def clean_newlines(contract_text: str) -> str: + """Clean textract output by removing newlines and page number indicators. + + This function processes input text to: + 1. Remove all newline characters. + 2. Remove lines that indicate page numbers (e.g. 'Page 1 of 10') + + Args: + contract_text (str): Raw text output from Textract to be cleaned + + Returns: + str: cleaned text with newlines and page number indicators removed + """ + cleaned_text = re.sub(r"(?