Merged in bugfix/dynamic-primary-issues (pull request #664)

Bugfix/dynamic primary issues * format and restructure * checkpoint * Merge branch 'main' into bugfix/dynamic-primary-issues * Update constants attributes to base class * Revert "Update constants attributes to base class" This reverts commit 936b1e41f2e2d445e69e840b69af08eac6661120. * Fix mapping * E2E test * remove prints * Merge branch 'main' into bugfix/dynamic-primary-issues * Black * Update any dynamic * Prompt update - more leniency * Remove prints * Merged main into bugfix/dynamic-primary-issues * update crosswalk * Merged main into bugfix/dynamic-primary-issues Approved-by: Alex Galarce
2025-08-14 16:46:28 +00:00
parent 105b5b107e
commit 8c99f5d073
15 changed files with 272 additions and 233 deletions
@@ -8,7 +8,7 @@
        "Preferred Provider Organization (PPO)": "PPO",
        "Exclusive Provider Organization (EPO)": "EPO",
        "Point of Service (POS)": "POS",
-        "PPO-POS" : "PPO | POS",
+        "PPO-POS" : "PPO|POS",
        "High Deductible Health Plan (HDHP)": "HDHP",
        "Indemnity" : "FFS"
    }
@@ -21,9 +21,9 @@
        "Temporary Assistance Needy Families (TANF)" : "TANF",
        "Long-Term Services and Supports (LTSS)" : "LTSS",
        "Special Needs Plan (SNP)" : "SNP",
-        "Medicare Advantage Special Needs Plan (MA-SNP)" : "MASNP",
-        "Chronic Condition Special Needs Plan (C-SNP)" : "CSNP",
-        "Dual Eligible Special Needs Plan (D-SNP)" : "DSNP",
+        "Medicare Advantage Special Needs Plan (MASNP)" : "MASNP",
+        "Chronic Condition Special Needs Plan (CSNP)" : "CSNP",
+        "Dual Eligible Special Needs Plan (DSNP)" : "DSNP",
        "Fully Integrated Dual Eligible Special Needs Plan (FIDE-SNP)" : "FIDE SNP",
        "Highly Integrated Dual Eligible Special Needs Plan (HIDE-SNP)" : "HIDE SNP",
        "Institutional Special Needs Plan (I-SNP)" : "ISNP",
@@ -73,13 +73,18 @@ def get_crosswalk_fields(answer_dicts):
            from_field_value = answer_dict.get(from_field)

            if not string_utils.is_empty(from_field_value):
-                to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping)
-
-                if not to_field_value or to_field_value == from_field_value:
+                if from_field_value in crosswalk_mapping.values():
+                    to_field_value = from_field_value
+                else:
                    to_field_value = apply_crosswalk(
-                        from_field_value, crosswalk_mapping_reverse
+                        from_field_value, crosswalk_mapping
                    )

+                    if not to_field_value or to_field_value == from_field_value:
+                        to_field_value = apply_crosswalk(
+                            from_field_value, crosswalk_mapping_reverse
+                        )
+
                answer_dict[to_field.field_name] = to_field_value
                just_mapped.append(to_field.field_name)

@@ -26,13 +26,29 @@ def prompt_dynamic(text, field_prompts, filename):
    return llm_answer_final


+def add_full_context_field(one_to_one_fields, field_to_add):
+    field_to_add.field_type = "full_context"
+    field_to_add.relationship = "one_to_one"
+
+    if field_to_add.allow_na:
+        field_to_add.prompt = (
+            field_to_add.prompt
+            + " Only answer if the answer applies to the ENTIRE contract. Otherwise, answer N/A."
+        )
+    else:
+        field_to_add.prompt = (
+            field_to_add.prompt
+            + " Do NOT leave this field N/A. Choose the most appropriate answer based on the context."
+        )
+    one_to_one_fields.add_field(field_to_add)
+    return one_to_one_fields
+
+
 def get_dynamic_one_to_one_fields(answer_dicts):
    """
    Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts
    """
-    fieldset = FieldSet(
-        config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True
-    )
+
    rate_escalator_fields = FieldSet(
        config.FIELD_JSON_PATH,
        relationship="one_to_n",
@@ -45,50 +61,59 @@ def get_dynamic_one_to_one_fields(answer_dicts):
        field_type="exhibit_level",
        field_name="OUTLIER_TERMS",
    )
-    fieldset.combine(rate_escalator_fields, inplace=True)
-    fieldset.combine(outlier_terms, inplace=True)

-    one_to_one_fields = FieldSet()
-    for field in fieldset.fields:
+    all_empty_fields = FieldSet(
+        config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True
+    )  # These fields get passed to 1:1 if ALL of the answers are empty
+    any_empty_fields = rate_escalator_fields.combine(
+        outlier_terms, inplace=False
+    )  # These fields get passed to 1:1 if ANY of the answers are empty
+    one_to_one_fields = FieldSet()  # Empty FieldSet to populate if criteria are met
+
+    # Handle ANY empty fields
+    for field in any_empty_fields.fields:
        field_name = field.field_name
-        if "REIMB" not in field_name:  # Don't do this for the REIMB_ fields
-            for answer_dict in answer_dicts:
-                if (
-                    string_utils.is_empty(answer_dict.get(field_name))
-                    or field_name not in answer_dict
-                ):
-                    if field_name == "RATE_ESCALATOR_DESC":
-                        # If the field is RATE_ESCALATOR_DESC, we need to add RATE_ESCALATOR_STATEMENT field and change its field type and relationship
-                        field_to_add = Field.load_from_file(
-                            file_path=config.FIELD_JSON_PATH,
-                            field_name="RATE_ESCALATOR_STATEMENT",
-                        )
-                    elif field_name == "OUTLIER_TERMS":
-                        field_to_add = Field.load_from_file(
-                            file_path=config.FIELD_JSON_PATH,
-                            field_name="OUTLIER_TERMS",
-                        )
-                    else:
-                        field_to_add = Field.load_from_file(
-                            file_path=config.FIELD_JSON_PATH,
-                            field_name=field.base_field,
-                        )
+        if "REIMB" in field_name:  # Don't do this for the REIMB_ fields
+            continue
+        for answer_dict in answer_dicts:
+            if string_utils.is_empty(answer_dict.get(field_name)):
+                if field_name == "RATE_ESCALATOR_DESC":
+                    # If the field is RATE_ESCALATOR_DESC, we need to add RATE_ESCALATOR_STATEMENT field and change its field type and relationship
+                    field_to_add = Field.load_from_file(
+                        file_path=config.FIELD_JSON_PATH,
+                        field_name="RATE_ESCALATOR_STATEMENT",
+                    )
+                elif field_name == "OUTLIER_TERMS":
+                    field_to_add = Field.load_from_file(
+                        file_path=config.FIELD_JSON_PATH,
+                        field_name="OUTLIER_TERMS",
+                    )
+                else:
+                    field_to_add = Field.load_from_file(
+                        file_path=config.FIELD_JSON_PATH,
+                        field_name=field.base_field,
+                    )
+                one_to_one_fields = add_full_context_field(
+                    one_to_one_fields, field_to_add
+                )
+                continue

-                    field_to_add.field_type = "full_context"
-                    field_to_add.relationship = "one_to_one"
-
-                    if field_to_add.allow_na:
-                        field_to_add.prompt = (
-                            field_to_add.prompt
-                            + " Only answer if the answer applies to the ENTIRE contract. Otherwise, answer N/A."
-                        )
-                    else:
-                        field_to_add.prompt = (
-                            field_to_add.prompt
-                            + " Do NOT leave this field N/A. Choose the most appropriate answer based on the context."
-                        )
-                    one_to_one_fields.add_field(field_to_add)
-                    continue
+    # Handle ALL empty fields
+    for field in all_empty_fields.fields:
+        field_name = field.field_name
+        if "REIMB" in field_name:  # Don't do this for the REIMB_ fields
+            continue
+        if all(
+            [
+                string_utils.is_empty(answer_dict.get(field_name))
+                for answer_dict in answer_dicts
+            ]
+        ):
+            field_to_add = Field.load_from_file(
+                file_path=config.FIELD_JSON_PATH,
+                field_name=field.base_field,
+            )
+            one_to_one_fields = add_full_context_field(one_to_one_fields, field_to_add)
    return one_to_one_fields


@@ -109,7 +134,11 @@ def prompt_dynamic_primary(exhibit_text, constants, field, filename):


 def get_dynamic_primary(
-    exhibit_chunk: str, exhibit_header: str, constants: Constants, filename: str, dynamic_fields: FieldSet
+    exhibit_chunk: str,
+    exhibit_header: str,
+    constants: Constants,
+    filename: str,
+    dynamic_fields: FieldSet,
 ):
    """
    Processes dynamic primary fields one-by-one from exhibit header and chunk text.
@@ -131,7 +160,13 @@ def get_dynamic_primary(
    reimbursement_level_fields = FieldSet()
    exhibit_level_answer_dict = {}
    for field in dynamic_fields.fields:
-        exhibit_header_answer = prompt_dynamic_primary(exhibit_header, constants, field, filename)
+        if string_utils.is_empty(exhibit_header):
+            exhibit_header_answer = "N/A"
+        else:
+            exhibit_header_answer = prompt_dynamic_primary(
+                exhibit_header, constants, field, filename
+            )
+        
        # If there is exactly ONE Non-N/A answer in the header
        if (
            not string_utils.is_empty(exhibit_header_answer)
@@ -152,7 +187,9 @@ def get_dynamic_primary(
            dynamic_fields.remove_field(field.field_name)
        # If there are ZERO Non-N/A answers in the header
        else:
-            exhibit_text_answer = prompt_dynamic_primary(exhibit_chunk, constants, field, filename)
+            exhibit_text_answer = prompt_dynamic_primary(
+                exhibit_chunk, constants, field, filename
+            )
            # If there is at least ONE Non-N/A answers in the Exhibit
            if not string_utils.is_empty(exhibit_text_answer):
                field.update_valid_values(exhibit_text_answer + " | N/A")
@@ -166,7 +203,11 @@ def get_dynamic_primary(


 def get_dynamic_answers(
-    exhibit_chunk: str, exhibit_header: str, filename: str, dynamic_fields: FieldSet, constants: Constants
+    exhibit_chunk: str,
+    exhibit_header: str,
+    filename: str,
+    dynamic_fields: FieldSet,
+    constants: Constants,
 ):
    """
    Processes dynamic fields from exhibit header and chunk text.
@@ -52,7 +52,12 @@ def process_file(file_object, constants: Constants, run_timestamp):

    ################## ONE TO ONE ##################
    one_to_one_results = run_one_to_one_prompts(
-        filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields, constants
+        filename,
+        contract_text,
+        text_dict,
+        top_sheet_dict,
+        dynamic_one_to_one_fields,
+        constants,
    )  # Return dict
    one_to_one_results["FILE_NAME"] = filename
    print(f"{datetime_str()} One to One Complete - {filename}")
@@ -64,6 +69,7 @@ def process_file(file_object, constants: Constants, run_timestamp):

    ################## CODES ##################
    final_results = code_funcs.code_breakout(merged_results, constants)
+    print(f"{datetime_str()} Codes Complete - {filename}")

    ################## POSTPROCESS ##################
    final_df = postprocess.postprocess(final_results, constants)
@@ -80,7 +86,12 @@ def process_file(file_object, constants: Constants, run_timestamp):


 def run_one_to_one_prompts(
-    filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields, constants
+    filename,
+    contract_text,
+    text_dict,
+    top_sheet_dict,
+    dynamic_one_to_one_fields,
+    constants,
 ):

    ################## INITIALIZE FIELDS ##################
@@ -114,7 +125,9 @@ def run_one_to_one_prompts(
    one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results)

    if "OUTLIER_TERMS" in one_to_one_results:
-        outlier_results = one_to_n_funcs.outlier_terms_breakout(one_to_one_results["OUTLIER_TERMS"],filename)
+        outlier_results = one_to_n_funcs.outlier_terms_breakout(
+            one_to_one_results["OUTLIER_TERMS"], filename
+        )
        one_to_one_results.update(outlier_results)

    return one_to_one_results[0]
@@ -133,89 +146,15 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants)
        exhibit_text = exhibit_dict[exhibit_page]

        ################# GET EXHIBIT-LEVEL ANSWERS ##################
-        if exhibit_page in all_exhibit_headers.keys():
-            exhibit_header = all_exhibit_headers[exhibit_page]
-        elif (
-            ".".join(exhibit_page.split(".")[:-1]) in all_exhibit_headers.keys()
-        ):  # Take the parent header if it exists
-            exhibit_header = all_exhibit_headers[".".join(exhibit_page.split(".")[:-1])]
-        else:
-            exhibit_header = "No Header Found"
-
-        exhibit_level_answers = one_to_n_funcs.get_exhibit_level_answers(
-            exhibit_text, constants, filename
-        )
-        exhibit_level_answers["EXHIBIT_PAGE"] = exhibit_page
-        exhibit_level_answers["EXHIBIT_TITLE"] = exhibit_header
-
-        # Extract exhibit lesser-of statement for passing to reimbursement_level
-        exhibit_lesser_of = exhibit_level_answers.get(
-            "EXHIBIT_LESSER_OF_STATEMENT", "N/A"
-        )
-
-        ################## INITIALIZE FIELDS ##################
-        reimbursement_level_fields = FieldSet(
-            relationship="one_to_n",
-            field_type="reimbursement_level",
-            file_path=FIELD_JSON_PATH,
-        )
-
-        ################## GET DYNAMIC-PRIMARY ANSWERS ##################
-        dynamic_to_exhibit_level_answers, dynamic_to_reimbursement_level_fields = (
-            dynamic_funcs.get_dynamic_primary(
-                exhibit_text,
-                exhibit_header,
-                constants,
-                filename,
-                FieldSet(
-                    relationship="one_to_n",
-                    field_type="dynamic_primary",
-                    file_path=config.FIELD_JSON_PATH,
-                ),
+        exhibit_level_answers, exhibit_lesser_of, exhibit_header = (
+            one_to_n_funcs.get_exhibit_level(
+                exhibit_text, exhibit_page, all_exhibit_headers, constants, filename
            )
        )
-        exhibit_level_answers.update(dynamic_to_exhibit_level_answers)
-        reimbursement_level_fields.combine(
-            dynamic_to_reimbursement_level_fields, inplace=True
-        )

-        ################## GET DYNAMIC-REIMB-INFO ANSWERS ##################
-        (
-            reimb_info_to_exhibit_level_answers,
-            reimb_info_to_reimbursement_level_fields,
-        ) = dynamic_funcs.get_dynamic_answers(
-            exhibit_text,
-            exhibit_header,
-            filename,
-            FieldSet(
-                relationship="one_to_n",
-                field_type="dynamic_reimb_info",
-                file_path=config.FIELD_JSON_PATH,
-            ),
-            constants
-        )
-        exhibit_level_answers.update(reimb_info_to_exhibit_level_answers)
-        reimbursement_level_fields.combine(
-            reimb_info_to_reimbursement_level_fields, inplace=True
-        )
-
-        ################## GET DYNAMIC-CODE ANSWERS ##################
-        code_to_exhibit_level_answers, code_to_reimbursement_level_fields = (
-            dynamic_funcs.get_dynamic_answers(
-                exhibit_text,
-                exhibit_header,
-                filename,
-                FieldSet(
-                    relationship="one_to_n",
-                    field_type="dynamic_code",
-                    file_path=config.FIELD_JSON_PATH,
-                ),
-                constants
-            )
-        )
-        exhibit_level_answers.update(code_to_exhibit_level_answers)
-        reimbursement_level_fields.combine(
-            code_to_reimbursement_level_fields, inplace=True
+        ################# GET DYNAMIC ANSWERS ##################
+        exhibit_level_answers, reimbursement_level_fields = one_to_n_funcs.get_dynamic(
+            exhibit_text, exhibit_header, exhibit_level_answers, constants, filename
        )

        ################## GET REIMBURSEMENT-LEVEL ANSWERS (INCLUDING DYNAMIC) ##################
@@ -16,6 +16,108 @@ from src import config
 from src.config import FIELD_JSON_PATH
 from src.prompts.prompt_templates import EXHIBIT_LEVEL
 from src.prompts.fieldset import FieldSet
+import src.investment.dynamic_funcs as dynamic_funcs
+
+
+def get_exhibit_level(
+    exhibit_text: str,
+    exhibit_page: str,
+    all_exhibit_headers: dict,
+    constants: Constants,
+    filename: str,
+):
+
+    if exhibit_page in all_exhibit_headers.keys():
+        exhibit_header = all_exhibit_headers[exhibit_page]
+    elif (
+        ".".join(exhibit_page.split(".")[:-1]) in all_exhibit_headers.keys()
+    ):  # Take the parent header if it exists
+        exhibit_header = all_exhibit_headers[".".join(exhibit_page.split(".")[:-1])]
+    else:
+        exhibit_header = "No Header Found"
+
+    exhibit_level_answers = get_exhibit_level_answers(exhibit_text, constants, filename)
+    exhibit_level_answers["EXHIBIT_PAGE"] = exhibit_page
+    exhibit_level_answers["EXHIBIT_TITLE"] = exhibit_header
+
+    # Extract exhibit lesser-of statement for passing to reimbursement_level
+    exhibit_lesser_of = exhibit_level_answers.get("EXHIBIT_LESSER_OF_STATEMENT", "N/A")
+
+    return exhibit_level_answers, exhibit_lesser_of, exhibit_header
+
+
+def get_dynamic(
+    exhibit_text: str,
+    exhibit_header: str,
+    exhibit_level_answers: dict,
+    constants: Constants,
+    filename: str,
+):
+
+    ################## INITIALIZE FIELDS ##################
+    reimbursement_level_fields = FieldSet(
+        relationship="one_to_n",
+        field_type="reimbursement_level",
+        file_path=FIELD_JSON_PATH,
+    )
+
+    ################## GET DYNAMIC-PRIMARY ANSWERS ##################
+    dynamic_to_exhibit_level_answers, dynamic_to_reimbursement_level_fields = (
+        dynamic_funcs.get_dynamic_primary(
+            exhibit_text,
+            exhibit_header,
+            constants,
+            filename,
+            FieldSet(
+                relationship="one_to_n",
+                field_type="dynamic_primary",
+                file_path=config.FIELD_JSON_PATH,
+            ),
+        )
+    )
+    exhibit_level_answers.update(dynamic_to_exhibit_level_answers)
+    reimbursement_level_fields.combine(
+        dynamic_to_reimbursement_level_fields, inplace=True
+    )
+
+    ################## GET DYNAMIC-REIMB-INFO ANSWERS ##################
+    (
+        reimb_info_to_exhibit_level_answers,
+        reimb_info_to_reimbursement_level_fields,
+    ) = dynamic_funcs.get_dynamic_answers(
+        exhibit_text,
+        exhibit_header,
+        filename,
+        FieldSet(
+            relationship="one_to_n",
+            field_type="dynamic_reimb_info",
+            file_path=config.FIELD_JSON_PATH,
+        ),
+        constants,
+    )
+    exhibit_level_answers.update(reimb_info_to_exhibit_level_answers)
+    reimbursement_level_fields.combine(
+        reimb_info_to_reimbursement_level_fields, inplace=True
+    )
+
+    ################## GET DYNAMIC-CODE ANSWERS ##################
+    code_to_exhibit_level_answers, code_to_reimbursement_level_fields = (
+        dynamic_funcs.get_dynamic_answers(
+            exhibit_text,
+            exhibit_header,
+            filename,
+            FieldSet(
+                relationship="one_to_n",
+                field_type="dynamic_code",
+                file_path=config.FIELD_JSON_PATH,
+            ),
+            constants,
+        )
+    )
+    exhibit_level_answers.update(code_to_exhibit_level_answers)
+    reimbursement_level_fields.combine(code_to_reimbursement_level_fields, inplace=True)
+
+    return exhibit_level_answers, reimbursement_level_fields


 def get_exhibit_level_answers(exhibit_chunk, constants, filename):
@@ -56,6 +158,7 @@ def get_reimbursement_primary(
 ) -> list[dict]:
    field_prompts = reimbursement_level_fields.print_prompt_dict(constants)
    prompt = prompt_templates.REIMBURSEMENT_LEVEL_PRIMARY(exhibit_text, field_prompts)
+
    logging.debug(
        f"""Running reimbursement primary prompt for {filename} with prompt:
 {prompt}"""
@@ -1165,9 +1268,7 @@ def process_special_cases(
            special_case_results.update(trigger_cap_results)
        elif case == "OUTLIER":
            logging.info(f"Running OUTLIERS breakout for {service} - {methodology}")
-            outlier_results = run_outliers_breakout(
-                service, methodology, filename
-            )
+            outlier_results = run_outliers_breakout(service, methodology, filename)
            special_case_results.update(outlier_results)
        else:  # Other/future special cases can go here
            pass
@@ -1225,6 +1326,7 @@ def run_trigger_cap_breakout(service: str, methodology: str, filename: str) -> d
    )
    return string_utils.universal_json_load(llm_response) or {}

+
 def run_outliers_breakout(service: str, methodology: str, filename: str) -> dict:
    """
    Runs focused OUTLIERS analysis using specialized prompt.
@@ -1241,17 +1343,14 @@ def run_outliers_breakout(service: str, methodology: str, filename: str) -> dict
        file_path=config.FIELD_JSON_PATH, field_type="outlier"
    ).print_prompt_dict()

-    prompt = prompt_templates.OUTLIER_BREAKOUT(
-        service, methodology, outlier_questions
-    )
+    prompt = prompt_templates.OUTLIER_BREAKOUT(service, methodology, outlier_questions)

    logging.debug(f"Running OUTLIERS breakout in {filename} with prompt: {prompt}")
    llm_response = llm_utils.invoke_claude(prompt, "sonnet_latest", filename)
-    logging.debug(
-        f"LLM Response for OUTLIERS breakout in {filename}: {llm_response}"
-    )
+    logging.debug(f"LLM Response for OUTLIERS breakout in {filename}: {llm_response}")
    return string_utils.universal_json_load(llm_response) or {}

+
 def outlier_terms_breakout(outlier_term, filename: str) -> dict:
    """Extract outlier fields from outlier term.

@@ -1261,8 +1360,10 @@ def outlier_terms_breakout(outlier_term, filename: str) -> dict:
    Returns:
        dict: A dictionary containing the extracted outlier terms.
    """
-    outlier_questions = FieldSet(file_path=config.FIELD_JSON_PATH, field_type="outlier").print_prompt_dict()
+    outlier_questions = FieldSet(
+        file_path=config.FIELD_JSON_PATH, field_type="outlier"
+    ).print_prompt_dict()
    prompt = prompt_templates.OUTLIER_TERMS_BREAKOUT(outlier_term, outlier_questions)
    llm_response = llm_utils.invoke_claude(prompt, "sonnet_latest", filename)
    outlier_answers = string_utils.universal_json_load(llm_response) or {}
-    return outlier_answers
+    return outlier_answers
@@ -102,7 +102,11 @@ def filter_payer_from_providers(


 def run_smart_chunked_fields(
-    one_to_one_fields: FieldSet, constants, contract_text: str, filename: str, text_dict: dict
+    one_to_one_fields: FieldSet,
+    constants,
+    contract_text: str,
+    filename: str,
+    text_dict: dict,
 ) -> dict[str, str]:
    """Process fields that require smart chunking and return the extracted answers.

@@ -383,9 +387,11 @@ def run_global_lesser_of_breakout(
    mock_service = "Global Lesser Of (all services)"

    prompt = REIMB_TERM_BREAKOUT(
-        mock_service, global_lesser_of_stmt, FieldSet(
-                    file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout"
-                ).print_prompt_dict(constants)
+        mock_service,
+        global_lesser_of_stmt,
+        FieldSet(
+            file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout"
+        ).print_prompt_dict(constants),
    )
    logging.debug(
        f"Running global lesser of breakout for {filename} with prompt: {prompt}"
@@ -4,7 +4,6 @@ from constants.investment_columns import COLUMN_ORDER
 from src.investment import postprocessing_funcs


-
 def postprocess(df, constants: Constants):
    if df.shape[0] > 0:

@@ -331,9 +331,7 @@ def link_exhibit_pages(
                exhibit_is_different = "Y"
                previous_header = page_header
            else:
-                prompt = prompt_templates.EXHIBIT_LINKAGE(
-                    page_header, previous_header
-                )
+                prompt = prompt_templates.EXHIBIT_LINKAGE(page_header, previous_header)
                claude_answer_raw = llm_utils.invoke_claude(
                    prompt, "legacy_sonnet", filename
                )
@@ -51,7 +51,10 @@ def merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results, constan


 def inject_global_lesser_of_rows(
-    one_to_n_results: pd.DataFrame, global_lesser_of_stmt: str, constants: Constants, filename: str
+    one_to_n_results: pd.DataFrame,
+    global_lesser_of_stmt: str,
+    constants: Constants,
+    filename: str,
 ) -> pd.DataFrame:
    """Create new rows with global lesser of constraint for REIMB_IDs that lack lesser of logic.

@@ -82,7 +85,7 @@ def inject_global_lesser_of_rows(
        else:
            one_to_n_results["LESSER_OF_IND"] = "N"
            rows_needing_injection = one_to_n_results.copy()
-        
+
    if rows_needing_injection.empty:
        logging.debug(f"No rows needing global lesser of injection for {filename}.")
        # Still need to set tracking flag for all existing rows
@@ -111,7 +111,9 @@ def stitch_chunks(chunks: list[str], overlap_size: int = CHUNK_OVERLAP) -> str:
    return result


-def field_context(one_to_one_fields: FieldSet, constants: Constants, contract_text: str, filename: str):
+def field_context(
+    one_to_one_fields: FieldSet, constants: Constants, contract_text: str, filename: str
+):
    """this function creates chunks from whole contract text and
    returns relevant chunks for each field based on sementic search and keyword search

@@ -1,8 +1,8 @@
-
 import json
 from constants.constants import Constants
 import src.utils.string_utils as string_utils

+
 class Field:
    def __init__(self, field_dict):
        # Base Attributes
@@ -94,17 +94,17 @@ class Field:
            f"Regex Pattern: {'None specified' if not self.regex_pattern else self.regex_pattern}"
        )

-    def get_prompt_dict(self, constants: Constants=None):
+    def get_prompt_dict(self, constants: Constants = None):
        """Returns a dictionary with the field name as the key and its prompt as the value."""
        return {self.field_name: self.get_prompt(constants)}

-    def get_prompt(self, constants: Constants=None):
+    def get_prompt(self, constants: Constants = None):
        """Resolves placeholders in the prompt using the valid_values property."""

        # If no valid_values to resolve
-        if string_utils.is_empty(self.valid_values): #
+        if string_utils.is_empty(self.valid_values):  #
            return self.prompt
-        
+
        if not string_utils.is_empty(self.valid_values) and not constants:
            raise AttributeError(
                "Must pass Constants object to resolve placeholders for fields with valid_values"
@@ -112,7 +112,7 @@ class Field:

        resolved_prompt = self.prompt
        if "{valid_values}" in resolved_prompt:
-            resolved_value = self.valid_values            
+            resolved_value = self.valid_values
            if hasattr(constants, resolved_value):
                try:
                    resolved_prompt = resolved_prompt.replace(
@@ -188,7 +188,9 @@ class FieldSet:
            if field.field_name == field_name:
                setattr(field, attr, value)

-    def get_prompt_dict(self, constants:Constants=None, field_type=None, relationship=None):
+    def get_prompt_dict(
+        self, constants: Constants = None, field_type=None, relationship=None
+    ):
        return {
            field.field_name: field.get_prompt(constants)
            for field in self.fields
@@ -196,9 +198,12 @@ class FieldSet:
            and (relationship is None or field.relationship == relationship)
        }

-    def print_prompt_dict(self, constants: Constants=None):
+    def print_prompt_dict(self, constants: Constants = None):
        return "\n".join(
-            [f"{field.field_name} : {field.get_prompt(constants)}" for field in self.fields]
+            [
+                f"{field.field_name} : {field.get_prompt(constants)}"
+                for field in self.fields
+            ]
        )

    def print(self):
@@ -213,7 +213,7 @@ Explain each answer before putting the final answer in the JSON dictionary forma


 def DYNAMIC_PRIMARY(context, field_name, field_prompt):
-    return f"""You are an information extractor. Your job is to extract attributes **strictly and literally** from the provided text, paying close attention to detail.
+    return f"""Extract attributes from the provided text, paying close attention to detail.

 [ATTRIBUTE TO EXTRACT]
 Here is the attribute, and instructions on how to correctly answer:
@@ -222,8 +222,8 @@ Here is the attribute, and instructions on how to correctly answer:
 [GENERAL INSTRUCTIONS]
 - Return all valid, explicitly stated values. If multiple values are found, list them separated by commas.
 - If none of the valid values appear explicitly in the text, return N/A.
- Never guess. Never assume. Only return values that are a direct, word-for-word match from the valid list.
- If a value appears in the text but is not in the list of valid values, ignore it.
+- Make appropriate and obvious assumptions, but don't take huge leaps of logic. 
+    - For example, you can assume that "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "CHIP Perinate", but do not assume that "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "Medicaid".

 Here is the text to analyze:

@@ -119,7 +119,6 @@ def yn_check(yn: str) -> tuple[str, bool]:
    return yn, (yn == "Y" or yn == "N")


-
 def pages_pagenum_check(pages: int, pagenum: int) -> bool:

    if not isinstance(pages, int):