diff --git a/fieldExtraction/constants/lists/exhibit_header_markers.json b/fieldExtraction/constants/lists/exhibit_header_markers.json index 7bf1614..4182040 100644 --- a/fieldExtraction/constants/lists/exhibit_header_markers.json +++ b/fieldExtraction/constants/lists/exhibit_header_markers.json @@ -5,5 +5,6 @@ "AMENDMENT", "SCHEDULE", "ADDENDUM", - "APPENDIX" + "APPENDIX", + "PARTICIPATING PROVIDER AGREEMENT" ] \ No newline at end of file diff --git a/fieldExtraction/src/investment/preprocessing_funcs.py b/fieldExtraction/src/investment/preprocessing_funcs.py index 4e682fe..4ed88ba 100644 --- a/fieldExtraction/src/investment/preprocessing_funcs.py +++ b/fieldExtraction/src/investment/preprocessing_funcs.py @@ -116,27 +116,19 @@ def get_exhibit_pages( sorted_pages = sorted(text_dict.keys(), key=string_utils.page_key_sort) exhibit_pages = [] exhibit_header_dict = {} - first_page = True for page_num in sorted_pages: page_content = text_dict[page_num] - # Always include the first page as an exhibit - if first_page: - exhibit_pages.append(page_num) - exhibit_header_dict[page_num] = "" - is_exhibit = True - first_page = False - else: - if "." not in page_num or ".0" in page_num: - exhibit_header = prompt_calls.prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename) - if "N/A" in exhibit_header: - is_exhibit = False - else: - exhibit_pages.append(page_num) - exhibit_header_dict[page_num] = exhibit_header - is_exhibit = True - elif is_exhibit == True: + if "." not in page_num or ".0" in page_num: + exhibit_header = prompt_calls.prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename) + if "N/A" in exhibit_header: + is_exhibit = False + else: exhibit_pages.append(page_num) + exhibit_header_dict[page_num] = exhibit_header + is_exhibit = True + elif is_exhibit == True: + exhibit_pages.append(page_num) return exhibit_pages, exhibit_header_dict @@ -330,6 +322,8 @@ def get_exhibit_dict(text_dict: dict, exhibit_chunk_mapping: dict) -> dict: # Group pages by exhibit exhibit_pages = {} for page, exhibit in exhibit_chunk_mapping.items(): + if exhibit.split(".")[0] == "0": + continue if exhibit not in exhibit_pages: exhibit_pages[exhibit] = [] exhibit_pages[exhibit].append(page)