jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/clean_full_context_in_smart_chunking (pull request #269)

Browse Source 
Bugfix/clean full context in smart chunking

* move full context bugfix to clean branch

* took out commented code

* black formating

* Add explanatory comments to file_processing.py

Approved-by: Alex Galarce
This commit is contained in:
Karan Desai
2024-11-13 21:33:17 +00:00
parent 001c2e45b5
commit dfff68e9c8
3 changed files with 152 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/preprocessing_funcs.py
-61
View File
@@ -265,64 +265,3 @@ def filter_quick_review(text_dict):
and "TOP SHEET" not in page_text.upper()
and "COVER SHEET" not in page_text.upper()
}
# Regex TIN Logic
# Regex search for all TINs (##-####### format)
# If exactly one TIN is found: that is the TIN
# If more than one TIN is found: chunk + prompt to find which of them is the correct TIN
# If zero TINs are found, regex search for all TINs (######### format)
# Then repeat steps 2 and 3 with the new pattern
# If still zero TINs are found, check for a TIN in the filename
# If still none, return empty
# def tin_regex(filename, text_dict: dict[str, str]):
# pattern_1 = r'\b\d{2}-?\d{7}\b'
# matches = []
# page_list = []
# for page in text_dict.keys():
# if len(re.findall((pattern_1, text_dict[page]))) != 0:
# matches.extend(re.findall((pattern_1, text_dict[page])))
# page_list.append(page)
# if len(matches) == 1:
# return matches[0]
# elif len(matches)>1:
# chunks = {}
# for pages in page_list:
# chunks[pages] = text_dict[pages]
# return chunks
# elif len(matches) == 0:
# pattern_2 = r'\b\d{9}\b'
# matches = []
# page_list = []
# for page in text_dict.keys():
# if len(re.findall((pattern_2, text_dict[page]))) != 0:
# matches.extend(re.findall((pattern_2, text_dict[page])))
# page_list.append(page)
# if len(matches) == 1:
# return matches[0]
# elif len(matches)>1:
# chunks = {}
# for pages in page_list:
# chunks[pages] = text_dict[pages]
# return chunks
# # find regex in file name
# tin = re.findall(pattern_1, filename)
# if len(tin) == 1:
# return tin[0]
# if len(tin) == 0:
# tin = re.findall(pattern_2, filename)
# if len(tin) == 1:
# return tin[0]
# return None