doczyai-pipelines/fieldExtraction/tests/test_code_funcs.py

import json
import re
import unittest
from unittest.mock import MagicMock, patch

import pandas as pd
import src.codes.code_funcs as code_funcs
from constants.constants import Constants
from constants.delimiters import Delimiter


class TestCodeFuncs(unittest.TestCase):

    def setUp(self):
        """Set up test fixtures before each test method."""
        # Create a mock Constants object
        self.constants = MagicMock(spec=Constants)

        # Configure mock constants
        self.constants.SYNONYM_MAP = {
            "DME": "DURABLE MEDICAL EQUIPMENT",
            "SNF": "SKILLED NURSING FACILITY",
        }
        self.constants.REMOVAL_LIST = ["INPATIENT", "OUTPATIENT", "SERVICES"]
        self.constants.STOP_WORD_LIST = ["THE", "AND", "OF"]
        self.constants.DO_NOT_RUN = ["BY REPORT", "STOP-LOSS"]

        # Setup mappings
        self.constants.CPT_LEVEL1_MAPPING = {"12345": "Test Procedure"}
        self.constants.HCPCS_LEVEL1_MAPPING = {"J0001": "Test Drug"}
        self.constants.REV_LEVEL1_MAPPING = {"123": "Test Revenue"}

        self.constants.CPT_LEVEL2_MAPPING = {"67890": "Advanced Procedure"}
        self.constants.HCPCS_LEVEL2_MAPPING = {"J0002": "Advanced Drug"}
        self.constants.REV_MAPPING = {"456": "Advanced Revenue"}

        self.constants.BILL_TYPE_MAPPING = {"11X": "Inpatient Hospital"}
        self.constants.BILL_TYPE_REVERSE_MAPPING = {"Inpatient Hospital": "11X"}

        # Mock the embedding model and function
        mock_embedding = MagicMock()
        mock_embedding.index.search.return_value = ([0.9], [[0]])
        mock_embedding.choices = ["Test Description"]
        self.constants.get_embedding.return_value = mock_embedding

        self.constants.EMBEDDING_MODEL = MagicMock()
        mock_encoded = MagicMock()
        mock_encoded.astype.return_value = "mock_vector"
        self.constants.EMBEDDING_MODEL.encode.return_value = mock_encoded

    def test_clean_service(self):
        """Test the clean_service function with various inputs."""
        # Test empty input
        self.assertEqual(code_funcs.clean_service("", self.constants), "")
        self.assertEqual(code_funcs.clean_service(None, self.constants), "")

        # Test synonym mapping
        self.assertEqual(
            code_funcs.clean_service("DME", self.constants), "DURABLE MEDICAL EQUIPMENT"
        )

        # Test removal list
        self.assertEqual(
            code_funcs.clean_service("INPATIENT SERVICES", self.constants), ""
        )

        # Test stop words only
        self.assertEqual(code_funcs.clean_service("THE AND OF", self.constants), "")

        # Test unlisted service
        self.assertEqual(
            code_funcs.clean_service("UNLISTED PROCEDURE", self.constants), "UNLISTED"
        )

        # Test normal case
        self.assertEqual(
            code_funcs.clean_service("CT SCAN SNF", self.constants),
            "CT SCAN SKILLED NURSING FACILITY",
        )

    @patch("src.utils.llm_utils.invoke_claude")
    @patch("src.utils.string_utils.universal_json_load")
    def test_code_explicit(self, mock_json_load, mock_invoke_claude):
        """Test the code_explicit function for extracting explicit codes."""
        # Setup mocks
        mock_invoke_claude.return_value = '{"PROCEDURE_CD": "12345"}'
        mock_json_load.return_value = {"PROCEDURE_CD": "12345"}

        # Test successful extraction
        result = code_funcs.code_explicit("TEST SERVICE","TEST_METHODOLOGY", "test.pdf")
        self.assertEqual(result, {"PROCEDURE_CD": "12345"})
        mock_invoke_claude.assert_called_once()

    @patch("src.utils.llm_utils.invoke_claude")
    @patch("src.utils.string_utils.universal_json_load")
    def test_code_category(self, mock_json_load, mock_invoke_claude):
        """Test the code_category function for detecting code categories."""
        # Setup mocks
        mock_invoke_claude.return_value = '["Drug A", "Drug B"]'
        mock_json_load.return_value = ["Drug A", "Drug B"]

        # Setup test data
        proc_category = ["Category: J"]
        hcpcs_mapping = {"J0001": "Drug A", "J0002": "Drug B", "K0001": "Other Drug"}

        # Test category matching
        result = code_funcs.code_category(
            "J CODES", proc_category, hcpcs_mapping, "test.pdf"
        )

        # Verify the results - code_category returns lists (JSON conversion happens in extract_codes_from_service)
        self.assertEqual(result["PROCEDURE_CD"], ["J0001", "J0002"])
        self.assertEqual(result["PROCEDURE_CD_DESC"], ["Drug A", "Drug B"])
        self.assertIn("J0001", result["PROCEDURE_CD"])
        self.assertIn("J0002", result["PROCEDURE_CD"])
        self.assertIn("Drug A", result["PROCEDURE_CD_DESC"])
        self.assertIn("Drug B", result["PROCEDURE_CD_DESC"])

    @patch("src.utils.llm_utils.invoke_claude")
    @patch("src.utils.string_utils.extract_text_from_delimiters")
    def test_code_implicit_special(self, mock_extract, mock_invoke_claude):
        """Test the code_implicit_special function for special case handling."""
        # Setup mocks
        mock_invoke_claude.return_value = "mock_response"
        mock_extract.return_value = "Drugs"

        # Test drug category - returns a list
        result = code_funcs.code_implicit_special("DRUG SERVICE", "test.pdf")
        self.assertEqual(result["PROCEDURE_CD"], ["J0000-J9999"])
        self.assertEqual(result["PROCEDURE_CD_DESC"], "Drugs")

        # Test no match
        mock_extract.return_value = ""
        result = code_funcs.code_implicit_special("OTHER SERVICE", "test.pdf")
        self.assertEqual(result, {})

    def test_get_embedding_levels(self):
        """Test the get_embedding_levels function for determining embedding levels."""
        # Test level 1 with both codes enabled
        implicit_run_dict = {"PROCEDURE_CD": True, "REVENUE_CD": True}
        embedding_levels, cpt_mapping, hcpcs_mapping, rev_mapping = (
            code_funcs.get_embedding_levels(1, implicit_run_dict, self.constants)
        )

        self.assertIn("cpt_level1", embedding_levels)
        self.assertIn("hcpcs_level1", embedding_levels)
        self.assertIn("rev_level1", embedding_levels)
        self.assertEqual(cpt_mapping, self.constants.CPT_LEVEL1_MAPPING)

        # Test level 2 with only procedure code enabled
        implicit_run_dict = {"PROCEDURE_CD": True, "REVENUE_CD": False}
        embedding_levels, cpt_mapping, hcpcs_mapping, rev_mapping = (
            code_funcs.get_embedding_levels(2, implicit_run_dict, self.constants)
        )

        self.assertIn("cpt_level2", embedding_levels)
        self.assertIn("hcpcs_level2", embedding_levels)
        self.assertNotIn("rev", embedding_levels)
        self.assertEqual(cpt_mapping, self.constants.CPT_LEVEL2_MAPPING)

    def test_get_match_list(self):
        """Test the get_match_list function for retrieving matches from embeddings."""
        # Configure mock for proper array-like return values
        mock_embedding = MagicMock()
        # Search returns a tuple of (scores, indices) where each is a 2D array
        mock_embedding.index.search.return_value = (
            # Scores array - 2D array with shape [1, top_k]
            [[0.9]],
            # Indices array - 2D array with shape [1, top_k]
            [[0]],
        )
        mock_embedding.choices = ["Test Description"]

        # Replace the mock embedding in constants for this test
        self.constants.get_embedding.return_value = mock_embedding

        # Test with constants mock configured in setUp
        match_list, highest_similarity = code_funcs.get_match_list(
            "TEST SERVICE", ["cpt_level1"], self.constants, 1
        )

        # Verify results
        self.assertEqual(match_list, ["Test Description"])
        self.assertEqual(highest_similarity, 0.9)

        # Verify the appropriate method calls
        self.constants.EMBEDDING_MODEL.encode.assert_called_once()
        self.constants.get_embedding.assert_called_once_with("cpt_level1")

    @patch("src.utils.llm_utils.invoke_claude")
    @patch("src.utils.string_utils.universal_json_load")
    @patch("src.codes.code_funcs.get_match_list")
    def test_code_implicit_rag(
        self, mock_get_match_list, mock_json_load, mock_invoke_claude
    ):
        """Test the code_implicit_rag function for RAG-based code extraction."""
        # Mock get_match_list to return properly formatted results
        mock_get_match_list.return_value = (["Test Procedure"], 0.9)

        # Setup mocks for Claude responses
        mock_invoke_claude.return_value = '["Test Procedure"]'
        mock_json_load.return_value = ["Test Procedure"]

        # Test with successful match
        implicit_run_dict = {"PROCEDURE_CD": True, "REVENUE_CD": True}

        # Patch the get_embedding_levels function to return predictable values
        with patch(
            "src.codes.code_funcs.get_embedding_levels"
        ) as mock_get_embedding_levels:
            # Configure mock to return appropriate values for embedding levels
            mock_get_embedding_levels.return_value = (
                ["cpt_level1"],
                {"12345": "Test Procedure"},
                {},
                {},
            )

            # Call the function
            result = code_funcs.code_implicit_rag(
                "TEST SERVICE", implicit_run_dict, "test.pdf", self.constants
            )

            # Verify results - code_implicit_rag returns lists
            self.assertEqual(result["PROCEDURE_CD"], ["12345"])
            self.assertEqual(result["PROCEDURE_CD_DESC"], ["Test Procedure"])
            self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Level 1")

        # Test exception handling
        mock_json_load.side_effect = Exception("Test error")
        result = code_funcs.code_implicit_rag(
            "TEST SERVICE", implicit_run_dict, "test.pdf", self.constants
        )
        self.assertIn("CODE_METHODOLOGY", result)

    @patch("src.utils.llm_utils.invoke_claude")
    @patch("src.utils.string_utils.extract_text_from_delimiters")
    def test_code_last_check(self, mock_extract, mock_invoke_claude):
        """Test the code_last_check function for categorizing non-matched services."""
        # Setup mocks
        mock_invoke_claude.return_value = "mock_response"

        # Test specific case
        mock_extract.return_value = "Specific"
        result = code_funcs.code_last_check("SPECIAL SERVICE", "test.pdf")
        self.assertEqual(result, "Specific")

        # Test generic case
        mock_extract.return_value = "Generic"
        result = code_funcs.code_last_check("GENERIC SERVICE", "test.pdf")
        self.assertEqual(result, "Generic")

    @patch("src.utils.llm_utils.invoke_claude")
    @patch("src.utils.string_utils.universal_json_load")
    def test_fill_bill_type(self, mock_json_load, mock_invoke_claude):
        """Test the fill_bill_type function for populating bill type information."""
        # Setup mocks
        mock_invoke_claude.return_value = '["Inpatient Hospital"]'
        mock_json_load.return_value = ["Inpatient Hospital"]

        # Test with valid bill type
        answer_dict = {}
        result = code_funcs.fill_bill_type(
            "INPATIENT SERVICE",
            answer_dict,
            self.constants.BILL_TYPE_MAPPING,
            self.constants.BILL_TYPE_REVERSE_MAPPING,
        )

        self.assertEqual(result["BILL_TYPE_CD"], "11X")
        self.assertEqual(result["BILL_TYPE_CD_DESC"], "Inpatient Hospital")

        # Test with empty response
        mock_json_load.return_value = []
        answer_dict = {"SERVICE_TERM": "Test"}
        result = code_funcs.fill_bill_type(
            "OTHER SERVICE",
            answer_dict,
            self.constants.BILL_TYPE_MAPPING,
            self.constants.BILL_TYPE_REVERSE_MAPPING,
        )

        self.assertEqual(result, {"SERVICE_TERM": "Test"})

    def test_get_implicit_runs(self):
        """Test the get_implicit_runs function for determining which code runs to execute."""
        # Test hospital claim type
        result = code_funcs.get_implicit_runs({"AARETE_DERIVED_CLAIM_TYPE_CD": "H"})
        self.assertTrue(result["REVENUE_CD"])

        # Test medical claim type with bill type
        result = code_funcs.get_implicit_runs(
            {"AARETE_DERIVED_CLAIM_TYPE_CD": "M", "BILL_TYPE_CD_DESC": "Outpatient"}
        )
        self.assertTrue(result["REVENUE_CD"])
        self.assertTrue(result["PROCEDURE_CD"])

        # Test inpatient bill type
        result = code_funcs.get_implicit_runs(
            {
                "AARETE_DERIVED_CLAIM_TYPE_CD": "M",
                "BILL_TYPE_CD_DESC": "Inpatient Hospital",
            }
        )
        self.assertTrue(result["REVENUE_CD"])
        self.assertFalse(result["PROCEDURE_CD"])

    @patch("src.codes.code_funcs.clean_service")
    @patch("src.codes.code_funcs.fill_bill_type")
    @patch("src.codes.code_funcs.get_implicit_runs")
    @patch("src.codes.code_funcs.code_explicit")
    @patch("src.codes.code_funcs.code_category")
    @patch("src.codes.code_funcs.code_implicit_special")
    @patch("src.codes.code_funcs.code_implicit_rag")
    @patch("src.codes.code_funcs.code_last_check")
    def test_extract_codes_from_service(
        self,
        mock_last_check,
        mock_implicit_rag,
        mock_implicit_special,
        mock_category,
        mock_explicit,
        mock_implicit_runs,
        mock_fill_bill_type,
        mock_clean_service,
    ):
        """Test the extract_codes_from_service function end-to-end."""
        # Setup mocks
        mock_clean_service.return_value = "CLEAN SERVICE"
        mock_fill_bill_type.return_value = {
            "SERVICE_TERM": "TEST SERVICE",
            "BILL_TYPE_CD_DESC": "Outpatient",
        }
        mock_implicit_runs.return_value = {"PROCEDURE_CD": True, "REVENUE_CD": True}

        # Test explicit code match - extract_codes_from_service normalizes to JSON list format
        mock_explicit.return_value = {"PROCEDURE_CD": "12345"}
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "TEST SERVICE"}, self.constants
        )
        self.assertEqual(result["PROCEDURE_CD"], '["12345"]')
        self.assertEqual(result["CODE_METHODOLOGY"], "Explicit")

        # Test unlisted service
        mock_clean_service.return_value = "UNLISTED"
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "UNLISTED SERVICE"}, self.constants
        )
        self.assertEqual(result["PROCEDURE_CD_DESC"], "UNLISTED")

        # Test category match
        mock_clean_service.return_value = "CLEAN SERVICE"
        mock_explicit.return_value = {"PROCEDURE_CD": ["Category: J"]}
        mock_category.return_value = {
            "PROCEDURE_CD": ["J0001"],
            "PROCEDURE_CD_DESC": ["Drug A"],
        }
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "J CODES"}, self.constants
        )
        self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Letter Category")

        # Test special case match
        mock_explicit.return_value = {}
        mock_implicit_special.return_value = {
            "PROCEDURE_CD": "J0000-J9999",
            "PROCEDURE_CD_DESC": "Drugs",
        }
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "DRUG SERVICE"}, self.constants
        )
        self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Special Case")

        # Test RAG match
        mock_implicit_special.return_value = {}
        mock_implicit_rag.return_value = {
            "PROCEDURE_CD": "12345",
            "CODE_METHODOLOGY": "Implicit - Level 1",
        }
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "TEST SERVICE"}, self.constants
        )
        self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Level 1")

        # Test no match - specific
        mock_implicit_rag.return_value = {}
        mock_last_check.return_value = "Specific"
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "TEST SERVICE"}, self.constants
        )
        self.assertEqual(result["CODE_METHODOLOGY"], "Specific - No Match")

        # Test no match - generic
        mock_last_check.return_value = "Generic"
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "TEST SERVICE"}, self.constants
        )
        self.assertEqual(result["CODE_METHODOLOGY"], "Generic - No Match")

    def test_fill_claim_type(self):
        """Test the fill_claim_type function for determining claim types."""
        # Test with existing claim types
        answer_dicts = [
            {"AARETE_DERIVED_CLAIM_TYPE_CD": "H"},
            {"AARETE_DERIVED_CLAIM_TYPE_CD": "H"},
            {"AARETE_DERIVED_CLAIM_TYPE_CD": "M"},
            {},
        ]
        result = code_funcs.fill_claim_type(answer_dicts)
        self.assertEqual(result[3]["AARETE_DERIVED_CLAIM_TYPE_CD"], "H")

        # Test with all empty claim types
        answer_dicts = [{}, {}]
        result = code_funcs.fill_claim_type(answer_dicts)
        self.assertEqual(result, answer_dicts)

    @patch("src.codes.code_funcs.fill_claim_type")
    @patch("src.codes.code_funcs.extract_codes_from_service")
    def test_code_breakout(self, mock_extract_codes, mock_fill_claim_type):
        """Test the code_breakout function for processing DataFrame records."""
        # Create test data
        test_df = pd.DataFrame(
            [{"SERVICE_TERM": "SERVICE A"}, {"SERVICE_TERM": "SERVICE B"}]
        )

        # Setup mocks
        mock_fill_claim_type.return_value = [
            {"SERVICE_TERM": "SERVICE A"},
            {"SERVICE_TERM": "SERVICE B"},
        ]

        mock_extract_codes.side_effect = [
            {"SERVICE_TERM": "SERVICE A", "PROCEDURE_CD": "12345"},
            {"SERVICE_TERM": "SERVICE B", "PROCEDURE_CD": "67890"},
        ]

        # Test function
        result_df = code_funcs.code_breakout(test_df, self.constants)

        # Verify results - code_breakout normalizes to JSON list format
        self.assertIsInstance(result_df, pd.DataFrame)
        self.assertEqual(len(result_df), 2)
        self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], '["12345"]')
        self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], '["67890"]')

        # Verify mock calls
        mock_fill_claim_type.assert_called_once()
        self.assertEqual(mock_extract_codes.call_count, 2)


if __name__ == "__main__":
    unittest.main()