diff --git a/.gitignore b/.gitignore index e7ee436..b8e278a 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,6 @@ __pycache__/ .env new/ + +# Embeddings +fieldExtraction/embeddings/ \ No newline at end of file diff --git a/fieldExtraction/src/investment/code_funcs.py b/fieldExtraction/src/investment/code_funcs.py index cec52a6..1cd4688 100644 --- a/fieldExtraction/src/investment/code_funcs.py +++ b/fieldExtraction/src/investment/code_funcs.py @@ -78,6 +78,60 @@ def crosswalk_levels(): return crosswalk_dict +def find_containing_ranges(input_range, code_mapping): + """ + Find which code ranges in the dictionary contain or overlap with the input range. + + Args: + input_range (str): Code range in format 'START-END' (e.g., '70010-79999') + code_mapping (dict): Dictionary containing CPT codes with code ranges as keys and descriptions as values + + Returns: + list: List of dictionaries containing matching code ranges and descriptions + """ + # Parse the input range + match = re.match(r'(\d+)-(\d+)', input_range) + if not match: + raise ValueError("Invalid input format. Expected format: 'START-END'") + + input_start = int(match.group(1)) + input_end = int(match.group(2)) + + # Initialize results list + results = [] + + # Process each code range in the dictionary + for code_range, description in code_mapping.items(): + # Parse the code range from the dictionary + range_match = re.match(r'(\d+)-(\d+)', code_range) + if range_match: + csv_start = int(range_match.group(1)) + csv_end = int(range_match.group(2)) + + # Check if there's any overlap between ranges + if max(input_start, csv_start) <= min(input_end, csv_end): + # The ranges overlap in some way + results.append({ + 'code_range': code_range, + 'description': description, + 'overlap_start': max(input_start, csv_start), + 'overlap_end': min(input_end, csv_end) + }) + # Extract only the code_range values + # code_ranges = [result['code_range'] for result in results] + # If no single range contains all others, return the highest level ranges + highest_level_ranges = [] + for result in results: + is_highest = True + for other in results: + if result != other and int(result['overlap_start']) >= int(other['overlap_start']) and int(result['overlap_end']) <= int(other['overlap_end']): + is_highest = False + break + if is_highest: + highest_level_ranges.append(result['code_range']) + return highest_level_ranges + # return code_ranges + def code_primary(answer_dicts, filename): """ Breaks down the direct code information from CONTRACT_SERVICE_CD_OR_DESC and CONTRACT_REIMBURSEMENT_METHOD fields @@ -266,29 +320,58 @@ def get_code_description(code, code_mapping): code_list = ast.literal_eval(code) elif "," in code: code_list = code.split(",") + elif "-" in code: + code_list = find_containing_ranges(code, code_mapping.mapping) else: - code_list = [code] + code_list = [code] # added this line to handle range strings like 'J0120-J7175' elif isinstance(code, list): code_list = code else: code_list = [code] - + code_description = [] for code in code_list: code_str = str(code).strip() description = code_mapping.mapping.get(code_str, "mismap") - + if string_utils.is_empty(description) or description == "mismap": - # If mismap, prepend "0" and try again (useful for rev codes) - code_str = "0" + code_str - description = code_mapping.mapping.get(code_str, "mismap") + if code_str.isdigit(): + description = code_mapping.mapping.get(int(code_str), "mismap") + else: + description = "mismap" + + # If mismap, try to handle ranges + if string_utils.is_empty(description) or description == "mismap": + if "X" in code_str: + base_code = code_str.replace("X", "") + expanded_code_range = f"{base_code}0-{base_code}9" + description = code_mapping.mapping.get(expanded_code_range, "mismap") + # If mismap, try to handle numeric codes + if string_utils.is_empty(description) or description == "mismap": + numeric_codes_description = [] + for i in range(10): + expanded_code = base_code + str(i) + inner_description = code_mapping.mapping.get(expanded_code, "mismap") + if inner_description != "mismap": + numeric_codes_description.append(inner_description) + description = ", ".join(numeric_codes_description) + else: + # If mismap, prepend "0" and try again (useful for rev codes) + code_str = "0" + code_str + description = code_mapping.mapping.get(code_str, "mismap") # We need similar logic for Diag codes with X (e.g. 123X) # Could either: a) convert to list ([1230, 1231, ...]), b) convert to range (1230-1239), c) leave as is (123X) but have the descriptions be a list ["desc1","desc2", etc] # for i in range(10): code is '123' + i' map that code if not string_utils.is_empty(description): - code_description.append(description) + code_description.append(description) + + # Remove repeated values and retain unique values, excluding "mismap" if other values are present + unique_descriptions = list(set(code_description)) + if "mismap" in unique_descriptions and len(unique_descriptions) > 1: + unique_descriptions.remove("mismap") + code_description = unique_descriptions return ",".join(code_description) diff --git a/fieldExtraction/tests/find_containing_ranges_test.py b/fieldExtraction/tests/find_containing_ranges_test.py new file mode 100644 index 0000000..31c7d11 --- /dev/null +++ b/fieldExtraction/tests/find_containing_ranges_test.py @@ -0,0 +1,59 @@ +import pytest +from unittest.mock import patch, MagicMock +from src.investment.code_funcs import find_containing_ranges + +def test_find_containing_ranges_valid_input(): + input_range = '70010-79999' + code_mapping = { + '70000-70050': 'Description 1', + '70040-70060': 'Description 2', + '70010-70020': 'Description 3', + '80000-80050': 'Description 4' + } + expected_output = ['70000-70050', '70040-70060'] + assert find_containing_ranges(input_range, code_mapping) == expected_output + +def test_find_containing_ranges_no_overlap(): + input_range = '70010-70020' + code_mapping = { + '70030-70040': 'Description 1', + '70050-70060': 'Description 2', + '80000-80050': 'Description 3' + } + expected_output = [] + assert find_containing_ranges(input_range, code_mapping) == expected_output + +def test_find_containing_ranges_partial_overlap(): + input_range = '70010-70050' + code_mapping = { + '70000-70020': 'Description 1', + '70040-70060': 'Description 2', + '70050-70070': 'Description 3' + } + expected_output = ['70000-70020', '70040-70060'] + assert find_containing_ranges(input_range, code_mapping) == expected_output + +def test_find_containing_ranges_exact_match(): + input_range = '70000-70050' + code_mapping = { + '70000-70050': 'Description 1', + '70040-70060': 'Description 2', + '70010-70020': 'Description 3' + } + expected_output = ['70000-70050'] + assert find_containing_ranges(input_range, code_mapping) == expected_output + +def test_find_containing_ranges_empty_mapping(): + input_range = '70000-70050' + code_mapping = {} + expected_output = [] + assert find_containing_ranges(input_range, code_mapping) == expected_output + +def test_find_containing_ranges_invalid_input(): + input_range = 'invalid-range' + code_mapping = { + '70000-70050': 'Description 1', + '70040-70060': 'Description 2' + } + with pytest.raises(ValueError): + find_containing_ranges(input_range, code_mapping) \ No newline at end of file