From 169bbf27733473a81b75b5e27fede8fa174b2df2 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 13 Oct 2025 19:03:31 +0000 Subject: [PATCH] Merged in bugfix/process-only-after-title (pull request #736) Bugfix/process only after title * Update print * Update exhibit dict creation * Detect Participating Provider Agreement * Remove test file * Merged main into bugfix/process-only-after-title * Merged main into bugfix/process-only-after-title Approved-by: Faizan Mohiuddin --- .../lists/exhibit_header_markers.json | 3 +- .../src/investment/preprocessing_funcs.py | 28 ++++++++----------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/fieldExtraction/constants/lists/exhibit_header_markers.json b/fieldExtraction/constants/lists/exhibit_header_markers.json index 7bf1614..4182040 100644 --- a/fieldExtraction/constants/lists/exhibit_header_markers.json +++ b/fieldExtraction/constants/lists/exhibit_header_markers.json @@ -5,5 +5,6 @@ "AMENDMENT", "SCHEDULE", "ADDENDUM", - "APPENDIX" + "APPENDIX", + "PARTICIPATING PROVIDER AGREEMENT" ] \ No newline at end of file diff --git a/fieldExtraction/src/investment/preprocessing_funcs.py b/fieldExtraction/src/investment/preprocessing_funcs.py index 4e682fe..4ed88ba 100644 --- a/fieldExtraction/src/investment/preprocessing_funcs.py +++ b/fieldExtraction/src/investment/preprocessing_funcs.py @@ -116,27 +116,19 @@ def get_exhibit_pages( sorted_pages = sorted(text_dict.keys(), key=string_utils.page_key_sort) exhibit_pages = [] exhibit_header_dict = {} - first_page = True for page_num in sorted_pages: page_content = text_dict[page_num] - # Always include the first page as an exhibit - if first_page: - exhibit_pages.append(page_num) - exhibit_header_dict[page_num] = "" - is_exhibit = True - first_page = False - else: - if "." not in page_num or ".0" in page_num: - exhibit_header = prompt_calls.prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename) - if "N/A" in exhibit_header: - is_exhibit = False - else: - exhibit_pages.append(page_num) - exhibit_header_dict[page_num] = exhibit_header - is_exhibit = True - elif is_exhibit == True: + if "." not in page_num or ".0" in page_num: + exhibit_header = prompt_calls.prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename) + if "N/A" in exhibit_header: + is_exhibit = False + else: exhibit_pages.append(page_num) + exhibit_header_dict[page_num] = exhibit_header + is_exhibit = True + elif is_exhibit == True: + exhibit_pages.append(page_num) return exhibit_pages, exhibit_header_dict @@ -330,6 +322,8 @@ def get_exhibit_dict(text_dict: dict, exhibit_chunk_mapping: dict) -> dict: # Group pages by exhibit exhibit_pages = {} for page, exhibit in exhibit_chunk_mapping.items(): + if exhibit.split(".")[0] == "0": + continue if exhibit not in exhibit_pages: exhibit_pages[exhibit] = [] exhibit_pages[exhibit].append(page)