diff --git a/fieldExtraction/crosswalk/crosswalk_utils.py b/fieldExtraction/crosswalk/crosswalk_utils.py index 5ef4451..fd328f1 100644 --- a/fieldExtraction/crosswalk/crosswalk_utils.py +++ b/fieldExtraction/crosswalk/crosswalk_utils.py @@ -171,4 +171,9 @@ def apply_crosswalk( Returns: str: Crosswalked value """ - return mapping.get(val, default if default is not None else val) + if "," in val: + values = val.split(",") + [val] # Split by comma if present + else: + values = [val] + mapped_values = [mapping.get(v.strip(), default if default is not None else v.strip()) for v in values] + return ",".join(mapped_values) # Join back into a single string diff --git a/fieldExtraction/crosswalk/mappings/crosswalk_provider_specialty.json b/fieldExtraction/crosswalk/mappings/crosswalk_provider_specialty.json index 2087830..beab897 100644 --- a/fieldExtraction/crosswalk/mappings/crosswalk_provider_specialty.json +++ b/fieldExtraction/crosswalk/mappings/crosswalk_provider_specialty.json @@ -4,128 +4,125 @@ "to_col": "CONTRACT_PROVIDER_SPECIALTY" }, "mapping": { - "01": "General Practice", - "02": "General Surgery", - "03": "Allergy/Immunology", - "04": "Otolaryngology", - "05": "Anesthesiology", - "06": "Cardiology", - "07": "Dermatology", - "08": "Family Practice", - "09": "Interventional Pain Management", - "10": "Gastroenterology", - "11": "Internal Medicine", - "12": "Osteopathic Manipulative Medicine", - "13": "Neurology", - "14": "Neurosurgery", - "15": "Speech Language Pathologist in Private Practice", - "16": "Obstetrics/Gynecology", - "17": "Hospice and Palliative Care", - "18": "Ophthalmology", - "19": "Oral Surgery (Dentists only)", - "20": "Orthopedic Surgery", - "21": "Cardiac Electrophysiology", - "22": "Pathology", - "23": "Sports Medicine", - "24": "Plastic and Reconstructive Surgery", - "25": "Physical Medicine and Rehabilitation", - "26": "Psychiatry", - "27": "Geriatric Psychiatry", - "28": "Colorectal Surgery (formerly Proctology)", - "29": "Pulmonary Disease", - "30": "Diagnostic Radiology", - "31": "Intensive Cardiac Rehabilitation (ICR)", - "32": "Anesthesiologist Assistant", - "33": "Thoracic Surgery", - "34": "Urology", - "35": "Chiropractic", - "36": "Nuclear Medicine", - "37": "Pediatric Medicine", - "38": "Geriatric Medicine", - "39": "Nephrology", - "40": "Hand Surgery", - "41": "Optometry", - "42": "Certified Nurse Midwife", - "43": "Certified Registered Nurse Anesthetist (CRNA)", - "44": "Infectious Disease", - "45": "Mammography Screening Center", - "46": "Endocrinology", - "47": "Independent Diagnostic Testing Facility (DTL)", - "48": "Podiatry", - "49": "Ambulatory Surgical Center", - "50": "Nurse Practitioner", - "59": "Ambulance Service Supplier", - "60": "Public Health/Welfare Agency", - "61": "Volunteer Health/Charitable Agency", - "62": "Psychologist (Billing Independently)", - "63": "Portable X-Ray Supplier (Billing Independently)", - "64": "Audiologist (Billing Independently)", - "65": "Physical Therapist in Private Practice", - "66": "Rheumatology", - "67": "Occupational Therapist in Private Practice", - "68": "Clinical Psychologist", - "69": "Clinical Laboratory (Billing Independently)", - "70": "Single or Multispecialty Clinic or Group Practice", - "71": "Registered Dietitian/Nutrition Professional", - "72": "Pain Management", - "73": "Mass Immunization Roster Biller", - "74": "Radiation Therapy Centers", - "75": "Slide Preparation Facilities", - "76": "Peripheral Vascular Disease", - "77": "Vascular Surgery", - "78": "Cardiac Surgery", - "79": "Addiction Medicine", - "80": "Licensed Clinical Social Worker", - "81": "Critical Care (Intensivist)", - "82": "Hematology", - "83": "Hematology/Oncology", - "84": "Preventative Medicine", - "85": "Maxillofacial Surgery", - "86": "Neuropsychiatry", - "88": "Unknown Provider", - "89": "Certified Clinical Nurse Specialist", - "90": "Medical Oncology", - "91": "Surgical Oncology", - "92": "Radiation Oncology", - "93": "Emergency Medicine", - "94": "Interventional Radiology", - "95": "Unknown Supplier", - "97": "Physician Assistant", - "98": "Gynecological Oncology", - "99": "Unknown Physician Specialty", - "A5": "Pharmacy", - "C0": "Sleep Medicine", - "C1": "Centralized Flu", - "C2": "Indirect Payment Procedure", - "C3": "Interventional Cardiology", - "C4": "Restricted Use", - "C5": "General Dentist", - "C6": "Hospitalist", - "C7": "Advanced Heart Failure and Transplant Cardiology", - "C8": "Medical Toxicology", - "C9": "Hematopoietic Cell Transplantation and Cellular Therapy", - "D1": "Medicare Diabetes Prevention Program", - "D2": "Restricted Use", - "D3": "Medical Genetics and Genomics", - "D4": "Undersea and Hyperbaric Medicine", - "D5": "Opioid Treatment Program", - "D6": "Home Infusion Therapy Services", - "D7": "Micrographic Dermatologic Surgery", - "D8": "Adult Congenital Heart Disease", - "E1": "Marriage and Family Therapist", - "E2": "Mental Health Counselors", - "E3": "Dental Anesthesiology", - "E4": "Dental Public Health", - "E5": "Endodontics", - "E6": "Oral and Maxillofacial Pathology", - "E7": "Oral and Maxillofacial Radiology", - "E9": "Oral Medicine", - "F1": "Orofacial Pain", - "F2": "Orthodontics and Dentofacial Orthopedics", - "F3": "Pediatric Dentistry", - "F4": "Periodontics", - "F5": "Prosthodontics", - "F6": "Epileptologists" - } - -} \ No newline at end of file + "GENERAL PRACTICE": "01", + "GENERAL SURGERY": "02", + "ALLERGY/IMMUNOLOGY": "03", + "OTOLARYNGOLOGY": "04", + "ANESTHESIOLOGY": "05", + "CARDIOLOGY": "06", + "DERMATOLOGY": "07", + "FAMILY PRACTICE": "08", + "INTERVENTIONAL PAIN MANAGEMENT": "09", + "GASTROENTEROLOGY": "10", + "INTERNAL MEDICINE": "11", + "OSTEOPATHIC MANIPULATIVE MEDICINE": "12", + "NEUROLOGY": "13", + "NEUROSURGERY": "14", + "SPEECH LANGUAGE PATHOLOGIST IN PRIVATE PRACTICE": "15", + "OBSTETRICS/GYNECOLOGY": "16", + "HOSPICE AND PALLIATIVE CARE": "17", + "OPHTHALMOLOGY": "18", + "ORAL SURGERY (DENTISTS ONLY)": "19", + "ORTHOPEDIC SURGERY": "20", + "CARDIAC ELECTROPHYSIOLOGY": "21", + "PATHOLOGY": "22", + "SPORTS MEDICINE": "23", + "PLASTIC AND RECONSTRUCTIVE SURGERY": "24", + "PHYSICAL MEDICINE AND REHABILITATION": "25", + "PSYCHIATRY": "26", + "GERIATRIC PSYCHIATRY": "27", + "COLORECTAL SURGERY (FORMERLY PROCTOLOGY)": "28", + "PULMONARY DISEASE": "29", + "DIAGNOSTIC RADIOLOGY": "30", + "INTENSIVE CARDIAC REHABILITATION (ICR)": "31", + "ANESTHESIOLOGIST ASSISTANT": "32", + "THORACIC SURGERY": "33", + "UROLOGY": "34", + "CHIROPRACTIC": "35", + "NUCLEAR MEDICINE": "36", + "PEDIATRIC MEDICINE": "37", + "GERIATRIC MEDICINE": "38", + "NEPHROLOGY": "39", + "HAND SURGERY": "40", + "OPTOMETRY": "41", + "CERTIFIED NURSE MIDWIFE": "42", + "CERTIFIED REGISTERED NURSE ANESTHETIST (CRNA)": "43", + "INFECTIOUS DISEASE": "44", + "MAMMOGRAPHY SCREENING CENTER": "45", + "ENDOCRINOLOGY": "46", + "INDEPENDENT DIAGNOSTIC TESTING FACILITY (DTL)": "47", + "PODIATRY": "48", + "AMBULATORY SURGICAL CENTER": "49", + "NURSE PRACTITIONER": "50", + "AMBULANCE SERVICE SUPPLIER": "59", + "PUBLIC HEALTH/WELFARE AGENCY": "60", + "VOLUNTEER HEALTH/CHARITABLE AGENCY": "61", + "PSYCHOLOGIST (BILLING INDEPENDENTLY)": "62", + "PORTABLE X-RAY SUPPLIER (BILLING INDEPENDENTLY)": "63", + "AUDIOLOGIST (BILLING INDEPENDENTLY)": "64", + "PHYSICAL THERAPIST IN PRIVATE PRACTICE": "65", + "RHEUMATOLOGY": "66", + "OCCUPATIONAL THERAPIST IN PRIVATE PRACTICE": "67", + "CLINICAL PSYCHOLOGIST": "68", + "CLINICAL LABORATORY (BILLING INDEPENDENTLY)": "69", + "SINGLE OR MULTISPECIALTY CLINIC OR GROUP PRACTICE": "70", + "REGISTERED DIETITIAN/NUTRITION PROFESSIONAL": "71", + "PAIN MANAGEMENT": "72", + "MASS IMMUNIZATION ROSTER BILLER": "73", + "RADIATION THERAPY CENTERS": "74", + "SLIDE PREPARATION FACILITIES": "75", + "PERIPHERAL VASCULAR DISEASE": "76", + "VASCULAR SURGERY": "77", + "CARDIAC SURGERY": "78", + "ADDICTION MEDICINE": "79", + "LICENSED CLINICAL SOCIAL WORKER": "80", + "CRITICAL CARE (INTENSIVIST)": "81", + "HEMATOLOGY": "82", + "HEMATOLOGY/ONCOLOGY": "83", + "PREVENTATIVE MEDICINE": "84", + "MAXILLOFACIAL SURGERY": "85", + "NEUROPSYCHIATRY": "86", + "UNKNOWN PROVIDER": "88", + "CERTIFIED CLINICAL NURSE SPECIALIST": "89", + "MEDICAL ONCOLOGY": "90", + "SURGICAL ONCOLOGY": "91", + "RADIATION ONCOLOGY": "92", + "EMERGENCY MEDICINE": "93", + "INTERVENTIONAL RADIOLOGY": "94", + "UNKNOWN SUPPLIER": "95", + "PHYSICIAN ASSISTANT": "97", + "GYNECOLOGICAL ONCOLOGY": "98", + "UNKNOWN PHYSICIAN SPECIALTY": "99", + "PHARMACY": "A5", + "SLEEP MEDICINE": "C0", + "CENTRALIZED FLU": "C1", + "INDIRECT PAYMENT PROCEDURE": "C2", + "INTERVENTIONAL CARDIOLOGY": "C3", + "GENERAL DENTIST": "C5", + "HOSPITALIST": "C6", + "ADVANCED HEART FAILURE AND TRANSPLANT CARDIOLOGY": "C7", + "MEDICAL TOXICOLOGY": "C8", + "HEMATOPOIETIC CELL TRANSPLANTATION AND CELLULAR THERAPY": "C9", + "MEDICARE DIABETES PREVENTION PROGRAM": "D1", + "MEDICAL GENETICS AND GENOMICS": "D3", + "UNDERSEA AND HYPERBARIC MEDICINE": "D4", + "OPIOID TREATMENT PROGRAM": "D5", + "HOME INFUSION THERAPY SERVICES": "D6", + "MICROGRAPHIC DERMATOLOGIC SURGERY": "D7", + "ADULT CONGENITAL HEART DISEASE": "D8", + "MARRIAGE AND FAMILY THERAPIST": "E1", + "MENTAL HEALTH COUNSELORS": "E2", + "DENTAL ANESTHESIOLOGY": "E3", + "DENTAL PUBLIC HEALTH": "E4", + "ENDODONTICS": "E5", + "ORAL AND MAXILLOFACIAL PATHOLOGY": "E6", + "ORAL AND MAXILLOFACIAL RADIOLOGY": "E7", + "ORAL MEDICINE": "E9", + "OROFACIAL PAIN": "F1", + "ORTHODONTICS AND DENTOFACIAL ORTHOPEDICS": "F2", + "PEDIATRIC DENTISTRY": "F3", + "PERIODONTICS": "F4", + "PROSTHODONTICS": "F5", + "EPILEPTOLOGISTS": "F6" + } +} diff --git a/fieldExtraction/src/constants/investment_values.py b/fieldExtraction/src/constants/investment_values.py index 06e0cd4..32d8eb6 100644 --- a/fieldExtraction/src/constants/investment_values.py +++ b/fieldExtraction/src/constants/investment_values.py @@ -1,4 +1,5 @@ +from crosswalk.crosswalk_utils import CrosswalkBuilder VALID_LOBS = [ "MEDICAID", @@ -15,6 +16,8 @@ VALID_LOBS = [ "TRICARE" ] +VALID_PROV_SPECIALTY = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_provider_specialty.json").mapping.keys()) + # Provider Type VALID_PROV_TYPE = [ "Physician", diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index 2723642..8ca68e3 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -30,17 +30,16 @@ def get_crosswalk_answers(crosswalk_fields, aarete_derived_answers_df): # Set up crosswalk crosswalk = CrosswalkBuilder() crosswalk_mapping = crosswalk.from_json(path=os.path.join("crosswalk/mappings", to_field.crosswalk)).mapping + #crosswalk_mapping_reverse = crosswalk.from_json(path=os.path.join("crosswalk/mappings", to_field.crosswalk)).create_reverse_mapping() # Determine the source column from_field = to_field.field_name if to_field.field_name in aarete_derived_answers_df else to_field.base_field if from_field in aarete_derived_answers_df: # Apply crosswalk transformation to the entire column - aarete_derived_answers_df[to_field.field_name] = ( - aarete_derived_answers_df[from_field].map(lambda x: apply_crosswalk(str(x).upper(), crosswalk_mapping)) + aarete_derived_answers_df[to_field.field_name] = aarete_derived_answers_df[from_field].map( + lambda x: apply_crosswalk(str(x).upper(), crosswalk_mapping) ) - # Right - return aarete_derived_answers_df diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 5f6dad2..07ac491 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -38,7 +38,7 @@ def process_file(file_object, all_dataset, run_timestamp): one_to_n_results['CONTRACT_FILE_NAME'] = filename print(f"{datetime_str()} One to N Complete - {filename}") else: - one_to_n_results = {'CONTRACT_FILE_NAME' : filename} + one_to_n_results = pd.DataFrame([{'CONTRACT_FILE_NAME' : filename}]) print(f"{datetime_str()} No One to N Found, Skipping - {filename}") ################## ONE TO ONE ################## @@ -117,12 +117,11 @@ def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mappi ################## GET REIMBURSEMENT-LEVEL ANSWERS (INCLUDING DYNAMIC) ################## reimbursement_level_answers = reimbursement_level(exhibit_chunk, filename, reimbursement_level_fields, all_dataset) # Return list of dictionaries - + ################## COMBINE ANSWERS ################## full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers) one_to_n_results += full_answer_dict - ################## CONVERT TO DF ################## one_to_n_df = pd.DataFrame(one_to_n_results) return one_to_n_df diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index e8bad37..d66a171 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -120,6 +120,7 @@ def dynamic_and_exhibit_level(exhibit_text, exhibit_header_results = prompt_dynamic(text=exhibit_header, field_prompts=dynamic_fields.get_prompt_dict(), filename=filename) + exhibit_level_answer_dict = process_dynamic(results=exhibit_header_results, reimbursement_level_fields=reimbursement_level_fields, exhibit_level_fields=exhibit_level_fields, @@ -361,8 +362,8 @@ def reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_ # Later, we will only run methodology breakout for those without certain carveout indicators. We will run carveout-specific breakouts for the others methodology_breakout_answers = get_methodology_breakout(carveout_answers, filename) - service_breakout_answers = get_service_breakout(methodology_breakout_answers, filename) - code_breakout_answers = code_funcs.get_code_breakout(service_breakout_answers, filename, all_dataset) + # service_breakout_answers = get_service_breakout(methodology_breakout_answers, filename) # Not using as of now - keep temporarily while field initialization is finishing. + code_breakout_answers = code_funcs.get_code_breakout(methodology_breakout_answers, filename, all_dataset) return code_breakout_answers else: diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index 08b31d8..a666014 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -335,14 +335,17 @@ { "field_name": "CONTRACT_PROV_SPECIALTY", "relationship": "one_to_n", - "field_type": "TBD", - "prompt": "TBD" + "field_type": "dynamic", + "prompt": "List any valid Provider Specialties that apply to the text. Choose ONLY from the following: {valid_values}", + "valid_values": "VALID_PROV_SPECIALTY" }, { "field_name": "AARETE_DERIVED_PROV_SPECIALTY", + "base_field": "CONTRACT_PROV_SPECIALTY", "relationship": "one_to_n", - "field_type": "TBD", - "prompt": "TBD" + "field_type": "crosswalk", + "prompt": "TBD", + "crosswalk": "crosswalk_provider_specialty.json" }, { "field_name": "CONTRACT_BILL_TYPE", diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index eb2883d..b01efc1 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -21,7 +21,7 @@ class Field: self.field_name = field_dict['field_name'] self.relationship = field_dict['relationship'] self.field_type = field_dict['field_type'] - self.prompt = field_dict['prompt'] + self.prompt = field_dict['prompt'] if "prompt" in field_dict else None self.valid_values = field_dict['valid_values'] if 'valid_values' in field_dict else None # AArete-Derived Attributes self.base_field = field_dict['base_field'] if 'base_field' in field_dict else None