From 1c700122e8372e1796ddb83ec26d256ad07e0abc Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Wed, 11 Jun 2025 20:54:48 +0000 Subject: [PATCH] Merged in feature/service-reimb-deduplication (pull request #568) Add (service, methodology) deduplication with accumulator * Add deduplication with accumulator * Merge remote-tracking branch 'origin/main' into feature/service-reimb-deduplication * Replace print statement with logging.debug for cross-exhibit duplicate detection * Fix and add tests for one_to_n_funcs * Merge remote-tracking branch 'origin/main' into feature/service-reimb-deduplication * Add seen_pairs parameter to reimbursement_level function in tests Approved-by: Katon Minhas --- .../src/investment/file_processing.py | 4 +- .../src/investment/one_to_n_funcs.py | 59 ++++++++- fieldExtraction/tests/dynamic_test.py | 3 +- fieldExtraction/tests/one_to_n_test.py | 117 +++++++++++++++++- 4 files changed, 175 insertions(+), 8 deletions(-) diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 7934ae1..a9c35e8 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -118,6 +118,8 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase ################## RUN PROMPTS ################## one_to_n_results = [] + seen_pairs = set() # Accumulator for tracking duplicate pairs of (`SERVICE_TERM`, `REIMB_TERM`) + for exhibit_page in exhibit_dict.keys(): exhibit_text = exhibit_dict[exhibit_page] @@ -157,7 +159,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase reimbursement_level_fields.combine(code_to_reimbursement_level_fields, inplace=True) ################## GET REIMBURSEMENT-LEVEL ANSWERS (INCLUDING DYNAMIC) ################## - reimbursement_level_answers = reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page) # Return list of dictionaries + reimbursement_level_answers = reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page, seen_pairs) # Return list of dictionaries ################# COMBINE ANSWERS ################## full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers) diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 84f906d..b4ea8d1 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -441,7 +441,7 @@ def combine_one_to_n_answers( def reimbursement_level( - exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page: str + exhibit_text, filename, reimbursement_level_fields, all_dataset, exhibit_page: str, seen_pairs: set ): """ Processes reimbursement-level fields. @@ -453,20 +453,69 @@ def reimbursement_level( reimbursement_level_fields (FieldSet): A FieldSet object containing field prompts. all_dataset (dict): # TODO: add docstring here exhibit_page (str): The page number of the exhibit being processed (for a unique identifier) + seen_pairs (set): A set accumulator to track pairs of SERVICE_TERM and REIMB_TERM to avoid duplicate processing. + This is currently across all exhibits, but could be modified to be per-superexhibit if needed. + (i.e. if a exhibit 23.0 [superexhibit 23.0] has subexhibits 23.0, 23.1, 23.2, etc., + then the pairs could be tracked per superexhibit.) This is NOT currently implemented. Returns: dict: The parsed LLM response as a list of dictionaries with unique identifiers """ + reimbursement_primary_answers = get_reimbursement_primary( reimbursement_level_fields, exhibit_text, filename ) # Returns list of dicts if reimbursement_primary_answers: # If any reimbursements found - carveout_answers = get_special_cases(reimbursement_primary_answers, filename) - code_breakout_answers = code_funcs.get_code_breakout( - carveout_answers, filename, all_dataset + # Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs + deduplicated_answers = deduplicate_with_accumulator( + reimbursement_primary_answers, seen_pairs, exhibit_page, filename ) - return code_breakout_answers + if deduplicated_answers: # Only process if we have unique pairs + carveout_answers = get_special_cases(deduplicated_answers, filename) + code_breakout_answers = code_funcs.get_code_breakout( + carveout_answers, filename, all_dataset + ) + return code_breakout_answers + else: + return [] # All pairs were duplicates, return empty list + else: return [] + +def deduplicate_with_accumulator( + answers: list[dict], seen_pairs: set, exhibit_page: str, filename: str + ) -> list[dict]: + """Remove service-reimbursement pairs that have been seen in previous exhibits. + + Args: + answers (list[dict]): List of dictionaries with SERVICE_TERM and REIMB_TERM keys. + seen_pairs (set): A set of previously seen (SERVICE_TERM, REIMB_TERM) pairs. + exhibit_page (str): The exhibit page number, for logging. + filename (str): The name of the file being processed, for logging + + Returns: + list[dict]: Deduplicated list of dictionaries. + """ + deduplicated = [] + duplicates_found = 0 + + for answer_dict in answers: + service_term = answer_dict.get("SERVICE_TERM", "").strip() + reimb_term = answer_dict.get("REIMB_TERM", "").strip() + + # Create a tuple key for comparison + pair_key = (service_term, reimb_term) + + if pair_key not in seen_pairs: + seen_pairs.add(pair_key) # Update accumulator + deduplicated.append(answer_dict) + else: + duplicates_found += 1 + logging.debug(f"Cross-exhibit duplicate found in {filename}, exhibit {exhibit_page}: {service_term} - {reimb_term}") + + if duplicates_found > 0: + logging.info(f"Removed {duplicates_found} duplicate pairs from exhibit {exhibit_page} in {filename}") + + return deduplicated \ No newline at end of file diff --git a/fieldExtraction/tests/dynamic_test.py b/fieldExtraction/tests/dynamic_test.py index eaa4b9d..e7196e3 100644 --- a/fieldExtraction/tests/dynamic_test.py +++ b/fieldExtraction/tests/dynamic_test.py @@ -142,10 +142,11 @@ class TestOneToN(unittest.TestCase): mock_fields = MagicMock() mock_dataset = {} exhibit_page = "1" + seen_pairs = set() # Execute result = one_to_n_funcs.reimbursement_level( - self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page + self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page, seen_pairs ) # Assert diff --git a/fieldExtraction/tests/one_to_n_test.py b/fieldExtraction/tests/one_to_n_test.py index cc7a36e..be03221 100644 --- a/fieldExtraction/tests/one_to_n_test.py +++ b/fieldExtraction/tests/one_to_n_test.py @@ -104,10 +104,11 @@ class TestOneToN(unittest.TestCase): mock_fields = MagicMock() mock_dataset = {} exhibit_page = "1" + seen_pairs = set() # Execute result = one_to_n_funcs.reimbursement_level( - self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page + self.exhibit_text, self.filename, mock_fields, mock_dataset, exhibit_page, seen_pairs ) # Assert @@ -117,5 +118,119 @@ class TestOneToN(unittest.TestCase): mock_special_cases.assert_called_once() mock_code_breakout.assert_called_once() + @patch('src.investment.one_to_n_funcs.get_reimbursement_primary') + def test_reimbursement_level_deduplication(self, mock_reimbursement_primary): + """Test that duplicate service-reimbursement pairs are properly filtered""" + # Setup - simulate getting duplicate pairs + mock_reimbursement_primary.return_value = [ + {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"}, + {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"}, + {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"} # Duplicate + ] + + mock_fields = MagicMock() + mock_dataset = {} + exhibit_page = "1" + seen_pairs = set() + + with patch('src.investment.one_to_n_funcs.get_special_cases') as mock_special_cases, \ + patch('src.investment.code_funcs.get_code_breakout') as mock_code_breakout: + + # Mock the downstream functions + mock_special_cases.side_effect = lambda x, _: x # Pass through + mock_code_breakout.side_effect = lambda x, _, __: x # Pass through + + # Execute + result = one_to_n_funcs.reimbursement_level( + self.exhibit_text, self.filename, mock_fields, mock_dataset, + exhibit_page, seen_pairs + ) + + # Assert - should only have 2 unique pairs + self.assertEqual(len(result), 2) + # Verify seen_pairs was updated + self.assertEqual(len(seen_pairs), 2) + self.assertIn(("Service A", "Rate A"), seen_pairs) + self.assertIn(("Service B", "Rate B"), seen_pairs) + + @patch('src.investment.one_to_n_funcs.get_reimbursement_primary') + def test_reimbursement_level_cross_exhibit_deduplication(self, mock_reimbursement_primary): + """Test that pairs seen in previous exhibits are filtered out""" + # Setup - pre-populate seen_pairs as if previous exhibit processed + seen_pairs = {("Service A", "Rate A")} + + # Current exhibit has same pair plus new one + mock_reimbursement_primary.return_value = [ + {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"}, # Should be filtered + {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"} # Should be kept + ] + + mock_fields = MagicMock() + mock_dataset = {} + exhibit_page = "2" + + with patch('src.investment.one_to_n_funcs.get_special_cases') as mock_special_cases, \ + patch('src.investment.code_funcs.get_code_breakout') as mock_code_breakout: + + mock_special_cases.side_effect = lambda x, _: x + mock_code_breakout.side_effect = lambda x, _, __: x + + # Execute + result = one_to_n_funcs.reimbursement_level( + self.exhibit_text, self.filename, mock_fields, mock_dataset, + exhibit_page, seen_pairs + ) + + # Assert - should only process the new pair + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["SERVICE_TERM"], "Service B") + # Verify both pairs are now in seen_pairs + self.assertEqual(len(seen_pairs), 2) + + def test_reimbursement_level_all_duplicates(self): + """Test behavior when all pairs are duplicates""" + seen_pairs = {("Service A", "Rate A"), ("Service B", "Rate B")} + + with patch('src.investment.one_to_n_funcs.get_reimbursement_primary') as mock_primary: + mock_primary.return_value = [ + {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"}, + {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"} + ] + + mock_fields = MagicMock() + mock_dataset = {} + exhibit_page = "3" + + # Execute + result = one_to_n_funcs.reimbursement_level( + self.exhibit_text, self.filename, mock_fields, mock_dataset, + exhibit_page, seen_pairs + ) + + # Assert - should return empty list, no downstream processing + self.assertEqual(result, []) + + def test_deduplicate_with_accumulator(self): + """Test the deduplication function directly""" + answers = [ + {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"}, + {"SERVICE_TERM": "Service B", "REIMB_TERM": "Rate B"}, + {"SERVICE_TERM": "Service A", "REIMB_TERM": "Rate A"}, # Duplicate + {"SERVICE_TERM": " Service C ", "REIMB_TERM": " Rate C "} # Test stripping + ] + + seen_pairs = set() + + result = one_to_n_funcs.deduplicate_with_accumulator( + answers, seen_pairs, "test_page", "test_file.pdf" + ) + + # Should have 3 unique pairs (duplicates removed) + self.assertEqual(len(result), 3) + self.assertEqual(len(seen_pairs), 3) + + # Verify stripping works + self.assertIn(("Service C", "Rate C"), seen_pairs) + if __name__ == '__main__': unittest.main()