Merged in refactor/add-doc-classification (pull request #745)
Add document classification * Add document classification * Update state-specific classification * Merge branch 'main' into refactor/add-doc-classification * Merge branch 'main' into refactor/add-doc-classification * functional for null state * standardize output name Approved-by: Karan Desai
This commit is contained in:
Katon Minhas
@@ -0,0 +1,88 @@
|
||||
|
||||
|
||||
def apply_rules_by_state(row, state):
|
||||
"""
|
||||
Check if final verdict should be set as false based on state-specific rules.
|
||||
Returns True if document should be excluded (final_verdict = False).
|
||||
"""
|
||||
if state is None:
|
||||
return False
|
||||
|
||||
filename = row['Contract Name'].upper() # Convert to uppercase for case-insensitive matching
|
||||
pages = row['total_pages']
|
||||
|
||||
# Define state-specific exclusion keywords
|
||||
state_rules = {
|
||||
'WI': {
|
||||
'max_pages': 75,
|
||||
'keywords': [
|
||||
'W9', 'COM - LTR', 'FORM - FORM', 'FORM - LIC', 'FORM - OWN',
|
||||
'FORM - DOCS', 'FORM - APP', 'FORM - CRED', 'FORM - PPA',
|
||||
'FORM - WPA', 'CRF', 'APPLICATION', 'CREDENTIALING',
|
||||
'CRED DOCUMENTS', 'CRED DOCUMENT', ' PIF ', ' COLL ', ' ROS ', ' HDO '
|
||||
]
|
||||
},
|
||||
'MA': {
|
||||
'max_pages': None,
|
||||
'keywords': [
|
||||
'CRF', 'FORM - CRED', 'CREDENTIALING', 'COM - LTR',
|
||||
'FORM - APP', 'APPLICATION'
|
||||
]
|
||||
},
|
||||
'MS': {
|
||||
'max_pages': None,
|
||||
'keywords': [
|
||||
'FORM - OWN', ' PIF ', 'FORM - LIC', 'FORM - DOCS', 'FORM PPA', ' COLL '
|
||||
],
|
||||
'special_rules': {
|
||||
'W9': 6 # W9 with exactly 6 pages
|
||||
}
|
||||
},
|
||||
'SC': {
|
||||
'max_pages': None,
|
||||
'keywords': [
|
||||
'FORM - FORM', 'FORM - OWN', 'FORM - LIC', 'COM - LTR'
|
||||
],
|
||||
'special_rules': {
|
||||
'W9': 6 # W9 with exactly 6 pages
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state = state.upper()
|
||||
if state not in state_rules:
|
||||
return False
|
||||
|
||||
rules = state_rules[state]
|
||||
|
||||
# Check if state has a max page limit
|
||||
if rules['max_pages'] is not None:
|
||||
# For states with page limits (like WI)
|
||||
if pages > rules['max_pages']:
|
||||
return True # Exclude if pages exceed limit
|
||||
|
||||
# For documents within page limit, check for exclusion keywords
|
||||
for keyword in rules['keywords']:
|
||||
if keyword in filename:
|
||||
return True
|
||||
|
||||
# Check special rules
|
||||
if 'special_rules' in rules:
|
||||
for keyword, required_pages in rules['special_rules'].items():
|
||||
if keyword in filename and pages == required_pages:
|
||||
return True
|
||||
else:
|
||||
# For states without page limits (like MA, MS, SC)
|
||||
# Check keywords regardless of page count
|
||||
for keyword in rules['keywords']:
|
||||
if keyword in filename:
|
||||
return True
|
||||
|
||||
# Check special rules
|
||||
if 'special_rules' in rules:
|
||||
for keyword, required_pages in rules['special_rules'].items():
|
||||
if keyword in filename and pages == required_pages:
|
||||
return True
|
||||
|
||||
# If no exclusion rules matched, don't exclude
|
||||
return False
|
||||
@@ -0,0 +1,86 @@
|
||||
import pandas as pd
|
||||
import re
|
||||
from src.investment.preprocessing_funcs import clean_newlines, split_text, clean_law_symbols
|
||||
from src.document_classification import doc_classification_funcs
|
||||
from src.utils import io_utils, string_utils
|
||||
import src.config as config
|
||||
|
||||
OUTPUT_FILE = f'Document-Classification-{config.BATCH_ID}.xlsx'
|
||||
|
||||
def main(state=None):
|
||||
|
||||
yes_keywords = ['Agreement', 'Amendment', 'Amend', 'Agmt', 'Agree', 'Agrmnt', 'Standard Clause Letter', 'BH Letter', 'Appendix', 'AMD', 'Notice', 'Contract']
|
||||
no_keywords = ['void', 'do not use', 'Internal memo', 'Disclosure', 'Disc of Owner', 'W9', 'W-9', 'Credential', 'Coversheet', 'Cvrsht', 'Thumbs.db', 'Questionnaire',' Cert ', 'Certificate', 'email', 'Exclusion', 'Change Letter', 'Termination Letter', 'Form', 'Provider Letter', '.xls', '.lnk', 'Taxpayer', 'Regulatory Deeming', 'Administrative Amendment', 'Administrative Agreement', 'Instructions', 'Adobe Sign', 'Notice of Legal', 'Receipt ',
|
||||
"Dear", "to whom it may concern"]
|
||||
|
||||
yes_patterns = [(kw, re.compile(re.escape(kw), re.IGNORECASE)) for kw in yes_keywords]
|
||||
no_patterns = [(kw, re.compile(re.escape(kw), re.IGNORECASE)) for kw in no_keywords]
|
||||
|
||||
record = []
|
||||
|
||||
files = io_utils.read_input()
|
||||
|
||||
for filename, contract_text in files.items():
|
||||
|
||||
contract_text = clean_newlines(contract_text)
|
||||
contract_text = clean_law_symbols(contract_text)
|
||||
text_dict = split_text(contract_text)
|
||||
|
||||
# --- search in filename ---
|
||||
yes_keywords_in_filename = [kw for kw, pattern in yes_patterns if pattern.search(filename)]
|
||||
no_keywords_in_filename = [kw for kw, pattern in no_patterns if pattern.search(filename)]
|
||||
|
||||
# --- first 5 pages ---
|
||||
threshold = min(5, len(text_dict))
|
||||
first_five_pages = "\n".join(text_dict[str(page)] for page in range(1, threshold + 1))
|
||||
|
||||
# --- search in body (first 5 pages only) ---
|
||||
yes_keywords_in_body = [kw for kw, pattern in yes_patterns if pattern.search(first_five_pages)]
|
||||
no_keywords_in_body = [kw for kw, pattern in no_patterns if pattern.search(first_five_pages)]
|
||||
|
||||
filename_verdict = len(yes_keywords_in_filename) > 0
|
||||
body_verdict = len(yes_keywords_in_body) > 0
|
||||
final_verdict = filename_verdict or body_verdict
|
||||
|
||||
record.append((
|
||||
filename,
|
||||
len(text_dict),
|
||||
yes_keywords_in_filename,
|
||||
no_keywords_in_filename,
|
||||
yes_keywords_in_body,
|
||||
no_keywords_in_body,
|
||||
filename_verdict,
|
||||
body_verdict,
|
||||
final_verdict,
|
||||
))
|
||||
|
||||
|
||||
columns = [
|
||||
'Contract Name',
|
||||
'total_pages',
|
||||
'yes_keywords_in_filename',
|
||||
'no_keywords_in_filename',
|
||||
'yes_keywords_in_body',
|
||||
'no_keywords_in_body',
|
||||
'filename_verdict',
|
||||
'body_verdict',
|
||||
'final_verdict'
|
||||
]
|
||||
|
||||
df = pd.DataFrame(record, columns=columns)
|
||||
|
||||
# final verdict
|
||||
df['final_verdict'] = df['filename_verdict'] | df['body_verdict']
|
||||
df.loc[df['total_pages']<=3,'final_verdict'] = False
|
||||
|
||||
# Apply state-specific filtering if state is provided
|
||||
if not string_utils.is_empty(state):
|
||||
exclusion_mask = df.apply(lambda row: doc_classification_funcs.apply_rules_by_state(row, state), axis=1)
|
||||
df.loc[exclusion_mask, 'final_verdict'] = False
|
||||
print(f"Applied state-specific filtering for {state}!")
|
||||
|
||||
df.to_excel(OUTPUT_FILE, index=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(state=config.STATE)
|
||||
@@ -63,6 +63,22 @@ def clean_law_symbols(contract_text):
|
||||
|
||||
return contract_text
|
||||
|
||||
def clean_newlines(contract_text: str) -> str:
|
||||
"""Clean textract output by removing newlines and page number indicators.
|
||||
|
||||
This function processes input text to:
|
||||
1. Remove all newline characters.
|
||||
2. Remove lines that indicate page numbers (e.g. 'Page 1 of 10')
|
||||
|
||||
Args:
|
||||
contract_text (str): Raw text output from Textract to be cleaned
|
||||
|
||||
Returns:
|
||||
str: cleaned text with newlines and page number indicators removed
|
||||
"""
|
||||
cleaned_text = re.sub(r"(?<!\n)\n(?!\n)", " ", contract_text)
|
||||
cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text)
|
||||
return cleaned_text
|
||||
|
||||
def filter_quick_review(text_dict):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user