Merged in bugfix/exhibit-header-detection (pull request #518)

Bugfix/exhibit header detection

* set up testing

* Increased context length, changed prompt to investment_prompts from preprocessing_prompts

* extracting the header

* minor changes

* Merged main into bugfix/exhibit-header-detection

* cleanup prints and tests


Approved-by: Alex Galarce

fieldExtraction/src/preprocessing_funcs.py

@@ -1,6 +1,6 @@
 import re
 
-from src.prompts import preprocessing_prompts
+from src.prompts import investment_prompts
 import src.utils.llm_utils as llm_utils
 import src.utils.string_utils as string_utils
 from src import config, keywords
@@ -225,9 +225,9 @@ def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> list[str]:
             exhibit_pages.append(page_num)
             first_page = False
         else:
-            prompt = preprocessing_prompts.EXHIBIT_CHECK(page[0:100])
+            prompt = investment_prompts.EXHIBIT_CHECK(page[0:200])
             claude_answer_raw = llm_utils.invoke_claude(
-                prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10 # TODO: low priority, try increasing max_tokens and maybe pass multiple pages in to reduce overall calls
+                prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=100 # TODO: low priority, try increasing max_tokens and maybe pass multiple pages in to reduce overall calls
             )
             claude_answer_extracted = string_utils.extract_text_from_delimiters(
                                 claude_answer_raw, Delimiter.PIPE

fieldExtraction/src/prompts/investment_prompts.py

@@ -256,6 +256,34 @@ class FieldSet:
 ################################## PROMPT-TEMPLATES #################################
 #####################################################################################
 
+def EXHIBIT_CHECK(page_start):
+  
+        return f"""Analyze the following contract excerpt and determine if it indicates the start of a section. Follow the following instructions:
+
+1. Look specifically for the words 'Exhibit', 'Article', 'Amendment', 'Schedule', 'Attachment', or 'Addendum'. These words indicate the start of a new section and are usually followed by a number or letter (e.g., 'Attachment A').
+
+2. These words should appear in a header-style format. This means:
+   - On their own line
+   - Surrounded by whitespace or formatting like a title
+   - Capitalized and not part of a longer sentence
+
+3. Do not treat the following as a section start:
+   - Legal references such as 'Section 85.113' or 'Article IX'
+   - Numbered clauses like '1.20', '7.1', or '11.1'
+   - Any sentence that simply mentions one of the keywords above
+
+4. The phrase 'Additional Provisions' does not indicate the start of a section.
+
+5. Return Y if the excerpt indicates the start of a section, N if not. Please justify your answer, but enclose your final answer in |pipes|.
+
+Here is the text to analyze:
+
+## START CONTRACT TEXT ##
+{page_start}
+## END CONTRACT TEXT ##
+
+State your answer according to the instructions above."""
+
 def EXHIBIT_HEADER(context):
     return f"""Analyze the following contract excerpt and extract the full header found.