Merged in feature/grouper-fields (pull request #658)

Feature/grouper fields

* prompt changes

* print statements removed

* merged with main

* merge issue fixed

* updated grouper prompts

* updated grouper fields in fields order

* updated breakout prompt

* Merged main into feature/grouper-fields

* reverted grouper cd prompt

* updated goruper severity prompt

* Merge remote-tracking branch 'origin/main' into feature/grouper-fields

* updated grouper breakout function

* error fixes

* Merged main into feature/grouper-fields

* Merged main into feature/grouper-fields

* updated grouper_pct_rate and grouper_base_rate

* removed empty array post processing fix

* Merged main into feature/grouper-fields

* post-processing for grouper rate modified

* updated grouper type prompt

* Merged main into feature/grouper-fields

* grouper version prompt updated

* Merged main into feature/grouper-fields

* OPPS as APC

* Merged main into feature/grouper-fields


Approved-by: Katon Minhas

fieldExtraction/src/codes/code_funcs.py

@@ -11,7 +11,6 @@ from constants.constants import Constants
 from constants.delimiters import Delimiter
 from src.prompts.fieldset import FieldSet
 
-
 def clean_service(service, constants: Constants) -> str:
     """
     Cleans the service string by removing unnecessary terms and formatting it for further processing.
@@ -615,13 +614,52 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants):
     # Fill empty AARETE_DERIVED_CLAIM_TYPE_CD with mode
     answer_dicts = fill_claim_type(merged_results.to_dict(orient="records"))
 
-    final_answer_dicts = []
+    answer_dicts_with_code = []
     for answer_dict in answer_dicts:
         code_answer_dict = extract_codes_from_service(answer_dict, constants)
         answer_dict.update(code_answer_dict)
+        answer_dicts_with_code.append(answer_dict)
+
+    return pd.DataFrame(answer_dicts_with_code)
+
+
+def grouper_breakout(results_with_code: pd.DataFrame):
+    """
+    Processes a DataFrame to extract and fill in grouper-related information in cases where we got a grouper_cd from code_breakout
+    but didn't get grouper fields from methodology + grouper breakout as in special case grouper was not identified and extracted.
+    Args:
+        results_with_code (pd.DataFrame): DataFrame containing the answer dictionaries with code fields
+    Returns:
+        pd.DataFrame: DataFrame with updated grouper fields.
+    """
+
+    # try grouper fields again if they are empty and grouper_cd is not empty
+    answer_dicts_with_code = results_with_code.to_dict(orient="records")
+    final_answer_dicts = []
+    GROUPER_QUESTIONS = FieldSet(
+        file_path=config.FIELD_JSON_PATH, field_type="grouper"
+    ).print_prompt_dict()
+    for answer_dict in answer_dicts_with_code:
+        if (
+            string_utils.is_empty(answer_dict.get("GROUPER_TYPE"))
+            and not string_utils.is_empty(answer_dict.get("GROUPER_CD"))
+        ):
+            # run groper breakout prompt
+            grouper_breakout_prompt = prompt_templates.GROUPER_BREAKOUT(
+                answer_dict.get("SERVICE_TERM", ""),
+                answer_dict.get("REIMB_TERM", ""),
+                GROUPER_QUESTIONS
+            )
+            claude_answer_raw = llm_utils.invoke_claude(
+                grouper_breakout_prompt, "sonnet_latest", ""
+            )
+            try:
+                grouper_answer = string_utils.universal_json_load(claude_answer_raw)
+            except Exception as e:
+                grouper_answer = {"GROUPER_TYPE": e}
+
+            answer_dict.update(grouper_answer)
         final_answer_dicts.append(answer_dict)
 
-    # Convert back to DataFrame
-    final_answer_df = pd.DataFrame(final_answer_dicts)
+    return pd.DataFrame(final_answer_dicts)
 
-    return final_answer_df