jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/dynamic_and_generalized_branch (pull request #361)

Browse Source 
feature/dynamic and generalized branch

* included txt files for nltk_data

* move nltk_data to src

* Fix last upload count

* Last upload count bugfix

* Fixed B processing

* Remove client-specific postprocessing

* split consolidate_output

* Run by file

* Fix output

* Reconfigure smart_chunk fields

* Add Full Context

* Fix merge conflicts

* Regex, Smart-Chunked, and Full working - not adding smart-chunked-->full when necessary

* Modernized run_full_context_fields()

* Switched set to list in field_context

* Move fields from smart_chunked to full_context as part of 'field_context' function

* Working version with placeholders

* Update poetry and pyproject

* Update s3 output

* Remove deprecated unit test

* Updated error messages

* Updated smart chunk ac name to one to one

* Update dependencies - end-to-end test for write s3 functional

* Add basic multithreading

* Send individual output to s3/local


Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
2025-01-27 19:39:23 +00:00
parent d345dd0ed3
commit 063c95d6ac
68 changed files with 779357 additions and 982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/preprocessing_funcs.py
+4 -96
View File
@@ -1,12 +1,13 @@
import re
from collections import defaultdict
from itertools import chain, count, groupby
from src.prompts import preprocessing_prompts
import src.utils.llm_utils as llm_utils
import src.utils.string_utils as string_utils
from src import config, keywords
from src.regex.regex_patterns import PIPE_PATTERN
from src import config, keywords, smart_chunking_funcs
from src import config, keywords
from src.enums.delimiters import Delimiter
def remove_page_indicators(contract_text: str) -> str:
@@ -170,99 +171,6 @@ def chunk_consecutive(text_dict, exhibit_pages):
return final_dict
def smart_chunk_ac(
text_dict: dict,
contract_text: str,
keyword_mappings: dict = keywords.GROUPED_KEYWORD_MAPPINGS,
) -> dict:
"""Performs a keyword search on textract outputs to retrieve relevant chunks for a given field.
Args:
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
contract_text(str): Contract Text
Raises:
ValueError: When the `key_dict` from keyword_mappings has a methodology that is not implemented.
Current options are `or` and `hierarchy`.
Returns:
dict: A dictionary keyed by field (defined by the keys in keyword_mappings) valued by
corresponding chunks of text (input strings delimited by newlines).
"""
field_pages = defaultdict(set) # TODO: remove this, it's unused
chunks = defaultdict(str)
full_context = "\n".join(
[f"Page {page}:\n{content}" for page, content in text_dict.items()]
)
for field_group, key_dict in keyword_mappings.items():
if key_dict["methodology"] == "or":
page_list = smart_chunking_funcs.chunk_or(
text_dict, key_dict["keywords"], key_dict["case_sensitive"]
)
keyword_page_dict = smart_chunking_funcs.keyword_search(
text_dict, key_dict["keywords"], key_dict["case_sensitive"]
)
elif key_dict["methodology"] == "hierarchy":
page_list = smart_chunking_funcs.chunk_hierarchical(
text_dict, key_dict["keywords"], key_dict["case_sensitive"]
)
keyword_page_dict = smart_chunking_funcs.keyword_search(
text_dict, key_dict["keywords"], key_dict["case_sensitive"]
)
elif key_dict["methodology"] == "and":
page_list = smart_chunking_funcs.chunk_and(
text_dict, key_dict["keywords"], key_dict["case_sensitive"]
)
keyword_page_dict = smart_chunking_funcs.keyword_search(
text_dict, key_dict["keywords"], key_dict["case_sensitive"]
)
elif key_dict["methodology"] == "regex":
page_list = smart_chunking_funcs.chunk_regex(text_dict, key_dict["regex"])
keyword_page_dict = smart_chunking_funcs.keyword_search(
text_dict, key_dict["keywords"], key_dict["case_sensitive"], True
)
elif key_dict["methodology"] == "include_exclude":
# Chunk pages based on included and excluded keywords
page_list = smart_chunking_funcs.chunk_with_include_exclude_keywords(
text_dict, key_dict["included_keywords"], key_dict["excluded_keywords"]
)
# Create a copy of keywords list by combining included and excluded keywords - this is required for keyword search
included_keywords = key_dict["included_keywords"]
excluded_keywords = key_dict["excluded_keywords"]
keywords = list(chain(included_keywords,excluded_keywords))
# Perform keyword search on the text dictionary - this returns a dictionary keyed by keyword and values are list of pages containing that keyword
# This is not currently used but is available for debugging purposes (or future use cases)
keyword_page_dict = smart_chunking_funcs.keyword_search(
text_dict, keywords, key_dict["case_sensitive"]
)
else:
page_list = list(text_dict.keys())
keyword_page_dict = {"All pages": list(text_dict.keys())}
raise ValueError(
"methodology must be 'hierarchy' or 'or' - using entire contract as chunk"
)
if field_group == "initial_page_group": #TODO: this needs to be removed? as it's not present in GROUPED_KEYWORD_MAPPINGS
keyword_page_dict["initial_pages"] = [1, 2]
# chunks[field_group] = concatenated page texts
if len(page_list) == len(text_dict.keys()):
chunks[field_group] = contract_text
else:
chunks[field_group] = "\n".join(
[text_dict[str(page_num)] for page_num in page_list]
)
chunks[field_group + "_pages"] = keyword_page_dict
chunks[field_group + "_methodology"] = key_dict["methodology"]
chunks[field_group + "_case"] = key_dict["case_sensitive"]
return chunks
def filter_quick_review(text_dict):
"""
Cover Sheets (aka Top Sheets or Quick Review pages) are pages stapled to the front of the contract that contain manually written summaries of the contract's contents.