Merged in feature/default-term-new-prompt (pull request #539)

default term new prompt and de-duplication before Methodology Breakout

* Refine prompts for default indicator methodology and enhance instructions for final reimbursement methodology output

* Enhance DEFAULT_INDICATOR prompt to clarify its role as a fallback reimbursement method

* Implement service-reimb tracking to prevent duplicates

* Refactor methodology breakout processing to reduce code duplication and improve clarity

* Remove debugging print statements and improve item ID formatting in reimbursement processing

* Merge remote-tracking branch 'origin/main' into feature/default-term-new-prompt

* Refactor methodology breakout processing to remove duplicate tracking and simplify logic

* Merged main into feature/default-term-new-prompt

* Merged main into feature/default-term-new-prompt

* rename _process_answer_dict to _process_carveout_or_breakout


Approved-by: Katon Minhas

fieldExtraction/src/investment/one_to_n_funcs.py

@@ -304,11 +304,13 @@ def methodology_breakout_single_row(
         prompt = methodology_prompt_template(
                 service, contract_reimbursement_method, methodology_breakout_questions
             )
+        # print(prompt) # Debugging line
         llm_response = llm_utils.invoke_claude(
             prompt,
             config.MODEL_ID_CLAUDE35_SONNET,
             filename,
         )
+        # print(llm_response) # Debugging line
         try:
             methodology_breakout_answers = string_utils.universal_json_load(
                 llm_response
@@ -388,38 +390,35 @@ def get_methodology_breakout(
             # This shouldn't happen but is in for defensive programming purposes
             print(f"Found a nested list instead of a dict, flattening {answer_dict}")
             for nested_dict in answer_dict:
-                if (
-                    nested_dict.get("CARVEOUT_IND") == "Y"
-                ):  # check if there is a CARVEOUT that would require a different methodology breakout
-                    methodology_breakout_answers.append(nested_dict)
-                else:  # If not a carveout, run normal methodology breakout
-                    methodology_breakout_answers.extend(
-                        methodology_breakout_single_row(
-                            nested_dict,
-                            METHODOLOGY_BREAKOUT_QUESTIONS,
-                            FS_BREAKOUT_QUESTIONS,
-                            filename,
-                        )
-                    )
-            continue
+                _process_carveout_or_breakout(nested_dict, methodology_breakout_answers, filename)
         else:  # normal processing for dict items
-            # print(f"Processing answer_dict (type {type(answer_dict)}):", answer_dict)
-            if (
-                answer_dict.get("CARVEOUT_IND") == "Y"
-            ):  # check if there is a CARVEOUT that would require a different methodology breakout
-                methodology_breakout_answers.append(answer_dict)
-            else:  # If not a carveout, run normal methodology breakout
-                methodology_breakout_answers.extend(
-                    methodology_breakout_single_row(
-                        answer_dict,
-                        METHODOLOGY_BREAKOUT_QUESTIONS,
-                        FS_BREAKOUT_QUESTIONS,
-                        filename,
-                    )
-                )
+            _process_carveout_or_breakout(answer_dict, methodology_breakout_answers, filename)
 
     return methodology_breakout_answers
 
+def _process_carveout_or_breakout(answer_dict: dict, methodology_breakout_answers: list, filename: str):
+    """Processes a single answer dictionary for methodology breakout.
+
+    Args:
+        answer_dict (dict): Dictionary containing reimbursement details.
+        methodology_breakout_answers (list): List to append results to.
+        filename (str): The filename used for LLM processing.
+
+    Returns:
+        bool: True if the item was processed, False if it was already seen.
+    """
+    if answer_dict.get("CARVEOUT_IND") == "Y":
+        methodology_breakout_answers.append(answer_dict)
+    else:  # If not a carveout, run normal methodology breakout
+        methodology_breakout_answers.extend(
+            methodology_breakout_single_row(
+                answer_dict,
+                METHODOLOGY_BREAKOUT_QUESTIONS,
+                FS_BREAKOUT_QUESTIONS,
+                filename,
+            )
+        )
+    return True
 
 def get_service_breakout(answer_dicts, filename):
     """

fieldExtraction/src/prompts/investment_prompts.json

@@ -427,7 +427,7 @@
         "field_name": "DEFAULT_IND",
         "relationship": "one_to_n",
         "field_type": "methodology_breakout",
-        "prompt": "DEFAULT_INDICATOR:\nIndicates whether this methodology serves as a default/fallback payment rate when no other payment methods apply. The default/fallback payment rate is typically a low rate and based on billed charges.\nOutput 'Y' if ANY of these conditions are met in the methodology:\n1. Specifies a rate that applies to:\nservices not listed in the fee schedule\nservices for which no other payment method is specified\nall other services\nservices not otherwise specified\n2. Explicitly labeled as:\n'default rate' or 'default payment'\nOutput 'N' if:\nThe methodology is specific to listed services/procedures\nNo default/fallback language is present"
+        "prompt": "DEFAULT_INDICATOR:\nIndicates whether this methodology serves as a default/fallback payment rate when no other payment methods apply. The default/fallback payment rate is typically a low rate and based on billed charges.  This is an 'end-of-the-line' reimbursement when no other methodologies apply.\n\nOutput 'Y' ONLY if the methodology meets BOTH these criteria:\n1. Contains language indicating it applies to services not covered elsewhere, such as:\n   - 'services not listed in the fee schedule'\n   - 'services for which no other payment method is specified'\n   - 'all other services'\n   - 'services not otherwise specified'\n2. Is clearly a secondary/fallback option, NOT the primary/base reimbursement method, as indicated by:\n   - Position in the contract (appears after primary methodology is defined)\n   - Lower reimbursement rate than the base methodology (typically 50-60% of charges)\n   - Explicit 'default' or 'fallback' terminology\n   - Applies to a minor subset of services (not the majority of contract services)\n\nOutput 'N' if ANY of these are true:\n- The methodology is the primary/base reimbursement approach, even if defined by exclusion\n- The methodology applies to specific listed services/procedures\n- The methodology offers standard rates comparable to other contract methodologies\n- No default/fallback language is present\n\nExample of TRUE default (Y): 'For any services not specifically listed in Exhibits A-D, reimbursement shall be 50% of billed charges.'\nExample of base methodology defined by exclusion (N): 'Covered Services not included in the Service Categories in Table 1 shall be reimbursed at 100% of the Medicare fee schedule.'"
     },
     {
         "field_name": "CPT4_PROC_CD",

fieldExtraction/src/prompts/investment_prompts.py

@@ -467,7 +467,7 @@ Here is the text to analyze and respond to:
 Service: {service.replace('"', "'")}
 Methodology: {methodology.replace('"', "'")}
 
-Write your JSON list below:
+Please explain your reasoning as you work through the document. After your explanation, include the heading "FINAL REIMBURSEMENT METHODOLOGY:" followed by a properly formatted JSON list with your final output.
 """
 
 def FEE_SCHEDULE_BREAKOUT(methodology, pct_rate, questions):