Katon Minhas
3bb6abbcef
feature/simplify b field chunking * Add chunk_by_exhibit function and unit tests * Refactor preprocess * Updated to run BU primary on entire Exhibit * Makes BU secondary compatible with exhibit chunk mapping * Added unit tests for get_exhibit_chunk * Trimmed TD funcs * Downstream changes to run_top_down * Removed conditional funcs * Merged main into feature/simplify-b-field-chunking * removed conditional funcs import * Merged main into feature/simplify-b-field-chunking * Fixed run_bottom_up_primary arguments * Fixed TD primary input * Fixed grouped keyword mappings * Remove filename tracking until postprocessing * Removed postprocessing for client columns * Removed col-based postprocessing * Merged main into feature/simplify-b-field-chunking * Fixed NoneType * Add docstrings * Updated unit tests for removal of 'Filename' * Updated comments Approved-by: Alex Galarce
101 lines
4.2 KiB
Python
101 lines
4.2 KiB
Python
import csv
|
|
|
|
import src.utils.table_utils as table_utils
|
|
from src import config, keywords, preprocessing_funcs
|
|
|
|
|
|
def clean_text(contract_text):
|
|
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
|
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
|
return contract_text
|
|
|
|
|
|
def split_text(contract_text):
|
|
"""
|
|
Splits the contract text into separate pages and filters pages needing quick review.
|
|
|
|
This function processes the input contract text by splitting it into individual pages
|
|
and then categorizes these pages. Pages representing a Quick Review or Cover Page section are separated from
|
|
the main text.
|
|
|
|
Parameters:
|
|
contract_text (str): The full text of the contract to be processed.
|
|
|
|
Returns:
|
|
tuple: A tuple containing three elements:
|
|
- dict: A dictionary with keys as page numbers (str) and values as the text of those pages.
|
|
- dict: A dictionary of pages that were identified Quick Review.
|
|
- int: The total number of pages in the contract text.
|
|
|
|
Notes:
|
|
- This function utilizes `preprocessing_funcs.split_text` to divide the contract text
|
|
into a dictionary with page numbers as keys.
|
|
- It uses `preprocessing_funcs.filter_quick_review` to separate out pages that need quick
|
|
review from the main text dictionary.
|
|
"""
|
|
text_dict = preprocessing_funcs.split_text(
|
|
contract_text
|
|
) # return a dictionary with keys - page_num (str), values as the page_text
|
|
num_pages = len(text_dict.keys())
|
|
text_dict, top_sheet_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
|
return text_dict, top_sheet_dict, num_pages
|
|
|
|
|
|
def one_to_n_exhibit_chunking(text_dict, filename):
|
|
"""
|
|
Identifies exhibit pages in a document and maps all text pages to their respective exhibits.
|
|
|
|
This function identifies pages that contain exhibit headers from a given document and then
|
|
maps each page of the document to the nearest preceding exhibit header.
|
|
|
|
Parameters:
|
|
text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages.
|
|
filename (str): The name of the file being processed
|
|
|
|
Returns:
|
|
tuple: A tuple containing two elements:
|
|
- list: A list of page numbers that are identified as starting new exhibits.
|
|
- dict: A dictionary mapping each page number to the exhibit it belongs to, with exhibit pages as keys.
|
|
|
|
Notes:
|
|
- This function utilizes `preprocessing_funcs.get_exhibit_pages` to identify pages that start new exhibits based on the document text and filename.
|
|
- It uses `preprocessing_funcs.chunk_by_exhibit` to map each page to its respective exhibit based on the identified exhibit pages.
|
|
"""
|
|
exhibit_pages = preprocessing_funcs.get_exhibit_pages(
|
|
text_dict, filename
|
|
) # All pages with exhibit headers
|
|
exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit(text_dict, exhibit_pages)
|
|
return exhibit_pages, exhibit_chunk_mapping
|
|
|
|
|
|
def clean_tables(text_dict, filename):
|
|
"""
|
|
In current state, this function is a wrapper for align_and_format_tables. Call any new table-related preprocessing funcs here
|
|
|
|
Parameters:
|
|
text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages.
|
|
filename (str): The name of the file being processed
|
|
|
|
Returns:
|
|
dict: text_dict, with the tables aligned and formatted
|
|
"""
|
|
text_dict = table_utils.align_and_format_tables(text_dict, filename)
|
|
return text_dict
|
|
|
|
|
|
def one_to_one_smart_chunking(text_dict, contract_text, keyword_mappings=keywords.GROUPED_KEYWORD_MAPPINGS):
|
|
"""
|
|
In current state, this function is a wrapper for smart_chunk_ac. Call any new smart-chunking-related preprocessing funcs here
|
|
|
|
Parameters:
|
|
text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages.
|
|
contract_text (str): The full text of the contract
|
|
keyword_mappings (dict): keywords.GROUPED_KEYWORD_MAPPINGS by default
|
|
|
|
Returns:
|
|
dict: ac_chunks
|
|
"""
|
|
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text, keyword_mappings)
|
|
return ac_chunks
|
|
|