Merged in bugfix/mi-reimb-terms (pull request #740)

Bugfix/mi reimb terms * test changes * logging added * Merge branch 'main' into bugfix/generic-reimb-terms * parsing fixes * fix missing reimbursements * make service terms unique by adding additional context * Merge branch 'main' into bugfix/mi-reimb-terms * remomoving test changes * remomoving test changes * remomoving test changes * test case for universal_json_load added Approved-by: Katon Minhas
2025-10-22 15:21:43 +00:00
parent 775d7eb1fb
commit 3027d8e1d5
4 changed files with 82 additions and 15 deletions
@@ -205,6 +205,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants)
                filename,
            )
        )
+        logging.debug(f"Exhibit Level Answers for {filename}, Page {exhibit_page}: {exhibit_level_answers}")

        ############################### Get Reimbursement-Level ###############################
        reimbursement_primary_answers, special_case_primary_answers = (
@@ -219,6 +220,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants)
                constants,
            )
        )
+        logging.debug(f"Reimbursement Primary Answers for {filename}, Page {exhibit_page}: {reimbursement_primary_answers}")

        ################################ Get Breakouts ###############################
        reimbursement_level_answers, special_case_breakout_answers = (
@@ -258,6 +258,7 @@ If no reimbursement terms are found, return the text "NO_REIMBURSEMENT_TERMS_FOU
    - Default to 'Covered Services' when no clear service is specified
    - Use full plan/program names when explicitly referenced (e.g. 'COMMERCIAL HMO', 'MEDICARE SELECT BENEFIT PROGRAM')
    - Include all relevant codes with services (revenue codes, procedure codes, etc.)
+    - When multiple similar service categories exist in the text, include distinguishing context in the SERVICE_TERM. Don't use generic labels like 'Covered Services' for multiple different service types—instead, specify the program, payer, or scope (e.g., 'Medicare Covered Services', 'Medicaid Covered Services', 'Emergency Covered Services') so each term is unambiguously unique.

 2. REIMB_TERM Completeness:
    - Include complete context (preambles, definitions, conditional statements, unit of measure, etc.)
@@ -196,24 +196,34 @@ def universal_json_load(string_dict: str):
    take the last one positionally.
    """
    # Try to match dictionary or list of dictionaries first
-    dict_matches = re.findall(r"\{.*?\}|\[\s*?\{.*?\}\s*?\]", string_dict, re.DOTALL)
+    # It will find any substring starting with { or [ and ending with } or ].
+    # dict_matches = re.findall(r"\{.*?\}|\[\s*?\{.*?\}\s*?\]", string_dict, re.DOTALL)
+    dict_matches = re.findall(r'(\[.*\]|\{.*?\})', string_dict, re.DOTALL)
    if dict_matches:
        matched_str = dict_matches[-1]  # Take the last match in the string
        try:
            return json.loads(matched_str)
-        except json.JSONDecodeError as e:
-            logging.error(f"Failed to parse JSON: {e}")
-            logging.error(f"Original string: {string_dict}")
-            logging.error(f"Traceback: {traceback.format_exc()}")
-            # Use placeholders to preserve JSON structure characters for CSV compatibility
-            csv_safe_string = (
-                string_dict.replace(",", "[COMMA]")
-                .replace("\n", "[NEWLINE]")
-                .replace('"', "[QUOTE]")
-            )
-            raise ValueError(
-                f"JSON parsing failed: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
-            ) from e
+        except json.JSONDecodeError:
+            # If it fails, it might be a malformed list (e.g., "[{...} text {...}]").
+            # As a fallback, find the LAST dictionary within that malformed string.
+            fallback_matches = re.findall(r'\{.*?\}', matched_str, re.DOTALL)
+            if fallback_matches:
+                last_dict_in_string = fallback_matches[-1]
+                try:
+                    return json.loads(last_dict_in_string)
+                except json.JSONDecodeError as e:
+                    logging.error(f"Failed to parse JSON in fallback: {e}")
+                    logging.error(f"Original string: {string_dict}")
+                    logging.error(f"Traceback: {traceback.format_exc()}")
+                    # Use placeholders to preserve JSON structure characters for CSV compatibility
+                    csv_safe_string = (
+                        string_dict.replace(",", "[COMMA]")
+                        .replace("\n", "[NEWLINE]")
+                        .replace('"', "[QUOTE]")
+                    )
+                    raise ValueError(
+                        f"JSON parsing failed in fallback: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
+                    ) from e

    # If no dict pattern found, try to match list of strings
    list_matches = re.findall(r"\[(.*?)\]", string_dict, re.DOTALL)
@@ -4,7 +4,8 @@ import pytest
 import src.utils as utils
 import src.utils.llm_utils as llm_utils
 from constants.delimiters import Delimiter
-from src.utils.string_utils import (contains_reimbursement,
+from src.utils.string_utils import (universal_json_load,
+                                    contains_reimbursement,
                                    count_reimbursements_in_exhibit,
                                    extract_signature_page,
                                    extract_text_from_delimiters, is_empty,
@@ -107,6 +108,59 @@ class TestStringUtils:
    def test_json_parsing_search(self, response_text, field_list, expected):
        assert json_parsing_search(response_text, field_list) == expected

+    @pytest.mark.parametrize(
+        "raw_text, expected",
+        [
+            (
+                'Reasoning ... ```json [{"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}]``` ... More text.',
+                [{"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}],
+            ),
+            (
+                'Reasoning ... answer {"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"} ... More text.',
+                {"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"},
+            ),
+            (
+                'Reasoning ... answer list [{"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}]',
+                [{"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}],
+            ),
+            (
+                'Reasoning ... single answer {"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}',
+                {"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"},
+            ),
+            (
+                'Reasoning ...{"a":"b"}... ```json [{"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}]```',
+                [{"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}],
+            ),
+            (
+                'Reasoning ...{"a":"b"}... answer {"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"}',
+                {"SERVICE_TERM": "Medicare Covered Services", "REIMB_TERM": "Billed Charges"},
+            ),
+            (
+                'Reasoning ...{"a":"b"}... ```json [{"SERVICE_TERM": [1,2,3], "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}]```',
+                [{"SERVICE_TERM": [1,2,3], "REIMB_TERM": "Billed Charges"}, {"SERVICE_TERM": "Medicaid Covered Services", "REIMB_TERM": "120% of Medicaid fee schedule"}],
+            ),
+            (
+                'Reasoning ...{"a":"b"}... answer {"SERVICE_TERM": [1,2,3], "REIMB_TERM": "Billed Charges"}',
+                {"SERVICE_TERM": [1,2,3], "REIMB_TERM": "Billed Charges"},
+            ),
+            # following test case with nested dict will fail (because we are using non greedy regex for dict) - kept for reference
+            # (
+            #     'Reasoning ... answer {"SERVICE_TERM": [1,2,3], "REIMB_TERM": {"1":"Billed Charges"}}',
+            #     {"SERVICE_TERM": [1,2,3], "REIMB_TERM": {"1":"Billed Charges"}},
+            # ),
+            # following test case with separate lists will fail (because we are using greedy regex for list) - kept for reference
+            # (
+            #     'Reasoning ...[]... answer {"SERVICE_TERM": [1,2,3], "REIMB_TERM": "Billed Charges"}',
+            #     {"SERVICE_TERM": [1,2,3], "REIMB_TERM": {"1":"Billed Charges"}},
+            # ),
+        ],
+    )
+    def test_universal_json_load(
+        self, raw_text, expected
+    ):
+        result = universal_json_load(raw_text)
+        assert result == expected
+
    @pytest.fixture
    def mock_invoke_claude(self, mocker):
        return mocker.patch.object(llm_utils, "invoke_claude")