Merged in feature/service-reimb-deduplication (pull request #568)

Add (service, methodology) deduplication with accumulator

* Add deduplication with accumulator

* Merge remote-tracking branch 'origin/main' into feature/service-reimb-deduplication

* Replace print statement with logging.debug for cross-exhibit duplicate detection

* Fix and add tests for one_to_n_funcs

* Merge remote-tracking branch 'origin/main' into feature/service-reimb-deduplication

* Add seen_pairs parameter to reimbursement_level function in tests


Approved-by: Katon Minhas

fieldExtraction/src/investment/file_processing.py

@@ -118,6 +118,8 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
     
     ################## RUN PROMPTS ##################
     one_to_n_results = []
+    seen_pairs = set() # Accumulator for tracking duplicate pairs of (`SERVICE_TERM`, `REIMB_TERM`)
+
     for exhibit_page in exhibit_dict.keys():
         
         exhibit_text = exhibit_dict[exhibit_page]
@@ -157,7 +159,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
         reimbursement_level_fields.combine(code_to_reimbursement_level_fields, inplace=True)
 
         ################## GET REIMBURSEMENT-LEVEL ANSWERS (INCLUDING DYNAMIC) ##################
-        reimbursement_level_answers = reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page) # Return list of dictionaries
+        reimbursement_level_answers = reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page, seen_pairs) # Return list of dictionaries
 
         ################# COMBINE ANSWERS ##################
         full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers)

fieldExtraction/src/investment/one_to_n_funcs.py

@@ -441,7 +441,7 @@ def combine_one_to_n_answers(
 
 
 def reimbursement_level(
-    exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page: str
+    exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page: str, seen_pairs: set
 ):
     """
     Processes reimbursement-level fields.
@@ -453,20 +453,69 @@ def reimbursement_level(
         reimbursement_level_fields (FieldSet): A FieldSet object containing field prompts.
         all_dataset (dict): # TODO: add docstring here
         exhibit_page (str): The page number of the exhibit being processed (for a unique identifier)
+        seen_pairs (set): A set accumulator to track pairs of SERVICE_TERM and REIMB_TERM to avoid duplicate processing.
+            This is currently across all exhibits, but could be modified to be per-superexhibit if needed.
+            (i.e. if a exhibit 23.0 [superexhibit 23.0] has subexhibits 23.0, 23.1, 23.2, etc., 
+            then the pairs could be tracked per superexhibit.)  This is NOT currently implemented.
 
     Returns:
         dict: The parsed LLM response as a list of dictionaries with unique identifiers
     """
+
     reimbursement_primary_answers = get_reimbursement_primary(
         reimbursement_level_fields, exhibit_text, filename
     )  # Returns list of dicts
 
     if reimbursement_primary_answers:  # If any reimbursements found
-        carveout_answers = get_special_cases(reimbursement_primary_answers, filename)
-        code_breakout_answers = code_funcs.get_code_breakout(
-            carveout_answers, filename, all_dataset
+        # Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs
+        deduplicated_answers = deduplicate_with_accumulator(
+            reimbursement_primary_answers, seen_pairs, exhibit_page, filename
         )
 
-        return code_breakout_answers
+        if deduplicated_answers: # Only process if we have unique pairs
+            carveout_answers = get_special_cases(deduplicated_answers, filename)
+            code_breakout_answers = code_funcs.get_code_breakout(
+                carveout_answers, filename, all_dataset
+            )
+            return code_breakout_answers
+        else:
+            return [] # All pairs were duplicates, return empty list
+
     else:
         return []
+
+def deduplicate_with_accumulator(
+        answers: list[dict], seen_pairs: set, exhibit_page: str, filename: str
+    ) -> list[dict]:
+    """Remove service-reimbursement pairs that have been seen in previous exhibits.
+
+    Args:
+        answers (list[dict]): List of dictionaries with SERVICE_TERM and REIMB_TERM keys.
+        seen_pairs (set): A set of previously seen (SERVICE_TERM, REIMB_TERM) pairs.
+        exhibit_page (str): The exhibit page number, for logging.
+        filename (str): The name of the file being processed, for logging
+
+    Returns:
+        list[dict]: Deduplicated list of dictionaries.
+    """
+    deduplicated = []
+    duplicates_found = 0
+
+    for answer_dict in answers:
+        service_term = answer_dict.get("SERVICE_TERM", "").strip()
+        reimb_term = answer_dict.get("REIMB_TERM", "").strip()
+
+        # Create a tuple key for comparison
+        pair_key = (service_term, reimb_term)
+
+        if pair_key not in seen_pairs:
+            seen_pairs.add(pair_key) # Update accumulator
+            deduplicated.append(answer_dict)
+        else:
+            duplicates_found += 1
+            logging.debug(f"Cross-exhibit duplicate found in {filename}, exhibit {exhibit_page}: {service_term} - {reimb_term}")
+        
+    if duplicates_found > 0:
+        logging.info(f"Removed {duplicates_found} duplicate pairs from exhibit {exhibit_page} in {filename}")
+
+    return deduplicated

fieldExtraction/tests/dynamic_test.py

@@ -142,10 +142,11 @@ class TestOneToN(unittest.TestCase):
         mock_fields = MagicMock()
         mock_dataset = {}
         exhibit_page = "1"
+        seen_pairs = set()
         
         # Execute
         result = one_to_n_funcs.reimbursement_level(
-            self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page
+            self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page, seen_pairs
         )
         
         # Assert

fieldExtraction/tests/one_to_n_test.py

@@ -104,10 +104,11 @@ class TestOneToN(unittest.TestCase):
         mock_fields = MagicMock()
         mock_dataset = {}
         exhibit_page = "1"
+        seen_pairs = set()
         
         # Execute
         result = one_to_n_funcs.reimbursement_level(
-            self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page
+            self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page, seen_pairs
         )
         
         # Assert
@@ -117,5 +118,119 @@ class TestOneToN(unittest.TestCase):
         mock_special_cases.assert_called_once()
         mock_code_breakout.assert_called_once()
 
+    @patch('src.investment.one_to_n_funcs.get_reimbursement_primary')
+    def test_reimbursement_level_deduplication(self, mock_reimbursement_primary):
+        """Test that duplicate service-reimbursement pairs are properly filtered"""
+        # Setup - simulate getting duplicate pairs
+        mock_reimbursement_primary.return_value = [
+            {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"},
+            {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"},
+            {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"}  # Duplicate
+        ]
+        
+        mock_fields = MagicMock()
+        mock_dataset = {}
+        exhibit_page = "1"
+        seen_pairs = set()
+        
+        with patch('src.investment.one_to_n_funcs.get_special_cases') as mock_special_cases, \
+            patch('src.investment.code_funcs.get_code_breakout') as mock_code_breakout:
+            
+            # Mock the downstream functions
+            mock_special_cases.side_effect = lambda x, _: x  # Pass through
+            mock_code_breakout.side_effect = lambda x, _, __: x  # Pass through
+            
+            # Execute
+            result = one_to_n_funcs.reimbursement_level(
+                self.exhibit_text, self.filename, mock_fields, mock_dataset, 
+                exhibit_page, seen_pairs
+            )
+            
+            # Assert - should only have 2 unique pairs
+            self.assertEqual(len(result), 2)
+            # Verify seen_pairs was updated
+            self.assertEqual(len(seen_pairs), 2)
+            self.assertIn(("Service A", "Rate A"), seen_pairs)
+            self.assertIn(("Service B", "Rate B"), seen_pairs)
+
+    @patch('src.investment.one_to_n_funcs.get_reimbursement_primary')
+    def test_reimbursement_level_cross_exhibit_deduplication(self, mock_reimbursement_primary):
+        """Test that pairs seen in previous exhibits are filtered out"""
+        # Setup - pre-populate seen_pairs as if previous exhibit processed
+        seen_pairs = {("Service A", "Rate A")}
+        
+        # Current exhibit has same pair plus new one
+        mock_reimbursement_primary.return_value = [
+            {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"},  # Should be filtered
+            {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"}   # Should be kept
+        ]
+        
+        mock_fields = MagicMock()
+        mock_dataset = {}
+        exhibit_page = "2"
+        
+        with patch('src.investment.one_to_n_funcs.get_special_cases') as mock_special_cases, \
+            patch('src.investment.code_funcs.get_code_breakout') as mock_code_breakout:
+            
+            mock_special_cases.side_effect = lambda x, _: x
+            mock_code_breakout.side_effect = lambda x, _, __: x
+            
+            # Execute
+            result = one_to_n_funcs.reimbursement_level(
+                self.exhibit_text, self.filename, mock_fields, mock_dataset, 
+                exhibit_page, seen_pairs
+            )
+            
+            # Assert - should only process the new pair
+            self.assertEqual(len(result), 1)
+            self.assertEqual(result[0]["SERVICE_TERM"], "Service B")
+            # Verify both pairs are now in seen_pairs
+            self.assertEqual(len(seen_pairs), 2)
+
+    def test_reimbursement_level_all_duplicates(self):
+        """Test behavior when all pairs are duplicates"""
+        seen_pairs = {("Service A", "Rate A"), ("Service B", "Rate B")}
+        
+        with patch('src.investment.one_to_n_funcs.get_reimbursement_primary') as mock_primary:
+            mock_primary.return_value = [
+                {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"},
+                {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"}
+            ]
+            
+            mock_fields = MagicMock()
+            mock_dataset = {}
+            exhibit_page = "3"
+            
+            # Execute
+            result = one_to_n_funcs.reimbursement_level(
+                self.exhibit_text, self.filename, mock_fields, mock_dataset, 
+                exhibit_page, seen_pairs
+            )
+            
+            # Assert - should return empty list, no downstream processing
+            self.assertEqual(result, [])
+
+    def test_deduplicate_with_accumulator(self):
+        """Test the deduplication function directly"""
+        answers = [
+            {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"},
+            {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"},
+            {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"},  # Duplicate
+            {"SERVICE_TERM": " Service C ", "REIMB_TERM": " Rate C "}  # Test stripping
+        ]
+        
+        seen_pairs = set()
+        
+        result = one_to_n_funcs.deduplicate_with_accumulator(
+            answers, seen_pairs, "test_page", "test_file.pdf"
+        )
+        
+        # Should have 3 unique pairs (duplicates removed)
+        self.assertEqual(len(result), 3)
+        self.assertEqual(len(seen_pairs), 3)
+        
+        # Verify stripping works
+        self.assertIn(("Service C", "Rate C"), seen_pairs)
+
 if __name__ == '__main__':
     unittest.main()