From 81f34edc8d37cad477fe3b1c7074ceba7b883ad7 Mon Sep 17 00:00:00 2001 From: FaizanM2000 Date: Mon, 3 Jun 2024 03:05:57 -0500 Subject: [PATCH] added new postprocessing and bottom up and top down merging. also carveouts --- .gitignore | 3 + src/carveouts.py | 138 +++++++++ src/config.py | 9 +- src/integration_testing.ipynb | 565 ++++++++++++++++++++++++++++++++++ src/postprocess.py | 27 ++ src/postprocessingfuncs.py | 95 ++++++ src/prompt_funcs.py | 3 +- src/prompts.py | 36 ++- src/test_notebook.ipynb | 359 --------------------- 9 files changed, 867 insertions(+), 368 deletions(-) create mode 100644 src/carveouts.py create mode 100644 src/integration_testing.ipynb create mode 100644 src/postprocess.py create mode 100644 src/postprocessingfuncs.py delete mode 100644 src/test_notebook.ipynb diff --git a/.gitignore b/.gitignore index 8a56daa..2b36f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ texts/ myenv/ *.env *.txt +subset/ +output/ +docs/ diff --git a/src/carveouts.py b/src/carveouts.py new file mode 100644 index 0000000..18a2bc0 --- /dev/null +++ b/src/carveouts.py @@ -0,0 +1,138 @@ +import pandas as pd +from difflib import SequenceMatcher +import re +import time +import prompts +import claude_funcs + +def get_closest_substring_match(val, valid_values): + """ Returns the first match from valid_values using a case-insensitive substring match. """ + if pd.isna(val): + return None + val = val.strip().upper() + for valid_value in valid_values: + if valid_value.upper() in val: + return valid_value + return None + +def get_best_carveout_from_claude(carveout_list, service): + print("using claude to get best carveout...") + prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE" + while True: + try: + response = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + return response.strip() + except Exception as e: + print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...") + time.sleep(60) + +def check_prov_type_similarity(prov_type, carveout): + print("checking similarity between prov_type and carveout with claude...") + prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'" + while True: + try: + response = claude_funcs.invoke_claude_3(prompt, max_tokens=10) + return response.strip() + except Exception as e: + print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...") + time.sleep(60) + +def label_services(filepath, carveout_list, output_filepath): + # Read the CSV file + df = pd.read_csv(filepath) + + # Select the first 50 rows of the DataFrame + df = df.head(500) + + # Define primary service terms + primary_terms = ['Covered Services', 'Inpatient Services', 'Outpatient Services', 'Physician Services', + 'Inpatient', 'Outpatient', 'Physician', 'Medical', 'Surgical', 'Diagnostic', 'Therapeutic'] + + # Initialize columns for the results + df['carveout_matched'] = '' + df['label'] = '' + df['IS_CARVEOUT'] = '' + + # Initialize a nested dictionary to track occurrences for each Filename and TD_LOB + occurrence_tracker = {} + + carveout_count = 0 + + # Iterate through the rows in the DataFrame + for index, row in df.iterrows(): + service = str(row['SERVICE']) + prov_type = str(row['PROV_TYPE']) if pd.notna(row['PROV_TYPE']) else '' + td_lob = row['TD_LOB'] + filename = row['Filename'] + + # Initialize the nested dictionary for the filename and TD_LOB if not present + if filename not in occurrence_tracker: + occurrence_tracker[filename] = {} + if td_lob not in occurrence_tracker[filename]: + occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms} + + # Check if the service is a primary or similar to a primary + primary = get_closest_substring_match(service, primary_terms) + if primary: + # Ensure the primary term is initialized in the occurrence tracker + if primary not in occurrence_tracker[filename][td_lob]: + occurrence_tracker[filename][td_lob][primary] = 0 + + # Determine the label based on the count of this primary term for the given Filename and TD_LOB + count = occurrence_tracker[filename][td_lob][primary] + if count == 0: + df.at[index, 'label'] = 'primary' + elif count == 1: + df.at[index, 'label'] = 'secondary' + elif count == 2: + df.at[index, 'label'] = 'tertiary' + else: + df.at[index, 'label'] = 'additional' + occurrence_tracker[filename][td_lob][primary] += 1 + df.at[index, 'IS_CARVEOUT'] = 'N' + else: + # If PROV_TYPE is blank, continue with carveout determination + best_carveout_response = get_best_carveout_from_claude(carveout_list, service) + matched_carveout = best_carveout_response # Use the model response directly + + df.at[index, 'carveout_matched'] = matched_carveout + + if matched_carveout: + # Check similarity between PROV_TYPE and carveout + prov_type_similarity_score = SequenceMatcher(None, prov_type, matched_carveout).ratio() + if prov_type_similarity_score < 0.8: + prov_type_similarity = check_prov_type_similarity(prov_type, matched_carveout) + else: + prov_type_similarity = "True" + + if re.search(r'\b(True|Yes)\b', prov_type_similarity, re.IGNORECASE): + if matched_carveout not in occurrence_tracker[filename][td_lob]: + print('claude returned a match between provider type and service') + occurrence_tracker[filename][td_lob][matched_carveout] = 0 + count = occurrence_tracker[filename][td_lob][matched_carveout] + if count == 0: + df.at[index, 'label'] = 'primary' + elif count == 1: + df.at[index, 'label'] = 'secondary' + elif count == 2: + df.at[index, 'label'] = 'tertiary' + else: + df.at[index, 'label'] = 'additional' + occurrence_tracker[filename][td_lob][matched_carveout] += 1 + df.at[index, 'IS_CARVEOUT'] = 'N' + else: + df.at[index, 'IS_CARVEOUT'] = 'Y' + else: + df.at[index, 'IS_CARVEOUT'] = 'Y' + + carveout_count += 1 + + # Save the updated DataFrame to a new CSV file after every 10 carveouts + if carveout_count % 10 == 0: + df.to_csv(output_filepath, index=False) + print(f"Saved progress after processing {carveout_count} carveouts.") + + # Final save of the updated DataFrame to a new CSV file + df.to_csv(output_filepath, index=False) + +label_services('service.csv', prompts.get_carveout_list(), 'carveouts50.csv') diff --git a/src/config.py b/src/config.py index 3c8e05f..418548a 100644 --- a/src/config.py +++ b/src/config.py @@ -3,6 +3,7 @@ from datetime import datetime import boto3 from llama_index.llms.bedrock import Bedrock +#pip install llama-index-llms-bedrock # General Settings TEST = False # True to run test prompt - just for testing model connection @@ -28,12 +29,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXNEUYUPUY" -AWS_SECRET_ACCESS_KEY="3HljGiz+dqWBtj+dkqs/syyxoguUoTahdLyceEeo" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEBUaCXVzLWVhc3QtMiJHMEUCIQCX9LcYh9A28cLNA/lDEHJ6ZqrulN0FampT1ptNyX+9JAIgIFrmDiZS5KfHbZWptuvveZt2s7x5r0xDwCU6t6EgBAEqjAMIjv//////////ARAAGgw2NjAxMzEwNjg3ODIiDOLRiVmy5kw46/5mzCrgAscMrf+1rEm7kpwpmd+BHjLNV4ggEy0oqQQ6z5Ny5BV6Ouo0tRmXqOqSaI5/YKXeKsRo8s4lrDNNq8qIK0GxpugvmbqguKYfEzKwyihGi5IVVb2JGL1nBkL495v0w5j1fuXqNwB7VCPkJyJCqxDHeehgtQ1AymuVAiRowkWmr7vVvFQyKKcGsBJlIzBZyguTtJhT2HPCp8IW9LhQMS2XBAa4xCnHqpPZSYIoKyp2zPPwi3pBKm+7eE3NGH7V7/Vrl6IL29zNkk5K/IeqAcyBzaFgaCPilr5N6lSMeKlkz8w0ls3guZeaEjDRf/Mz3NvMD9HeUVqGfi2M+ODxudgEfmfH3Op0EC2g5fsIBu/uvcjJf7PgUo9xT6CH5P5HY/e6TLaZH55ioz468uaTfoNw10TZ/6iymqYEXSpLfqHejC6qCx2pfcYf1UizgD2rn22GfcGs8u6oKFSThJ4fETq1mogwzafCsgY6pgHl1EzoiRwiXqCDl3GsA162dYrapmEPqcIVXTTuNuYXbl7B3GQfxTmjHjqSqKFhieWEvydD4SSQAhSXz9tPsN69Myyelwg3t6r2EWGL+JiZXEFpO1peWJgu3xfd3F1tPnp7Nfv7bVGmuIvft1JYTmx20W/vH9Q4J7jM4tPqEeQs5fRORTdXcm10Yysuq5FYo+9bahUJnPYblcJveO30vVirwzVWvmPm" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFULN3ZDJ" +AWS_SECRET_ACCESS_KEY="biHj+LwFkcquA7hoOgWst66u7AAQcik+A8TnMSoO" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELr//////////wEaCXVzLWVhc3QtMiJHMEUCIEIZfQMj70zYiV1czvy0RdOdugmJ9YGZIPBk/wqUE8m1AiEAzA9jGoZfv0ZMZlBKTLHGT3Ax3rAQNVLYWqgo9jy3Ui8qiQMIQxAAGgw2NjAxMzEwNjg3ODIiDOYEfYdB5ykwPYhKOSrmAiHPCbfUR2lXoqyEvURpTFyOqLEmeVFHCHQoVyeFcXwQmjjZor+ui1dclq0asj35ZSScClpyQKZsfk5o5+76pxzTm1tlBez1buIFkel/9OngHNG4MvMwLCMLdI+ILtJy0ufLmky4oVgtSor5n2e5dLENqstqhXEREbGdxZ5gaF7zKg9Dk/VPQ8arzV0oEvxuPGbZ+Z640jtVR27oSU1pQQ4ayK+CM/jBMANlzO9BptRB+Phfm6oOfxYQskyIXRV+aff/3d/MVJs6X08U/YjN+qXx0pDLsyBxYQIpKZXIZvWrYRD/NM7DnPJF8DlTAWhNSDU0QJ4j2oTMKcaG0LHXmd1SejjgHXhK0ZiG2tcpNHPaIBFm/lWnaixOdxM9XUWckoTe+1FRmcblQ7YI2HtWsbgzU3B8BLmLvEPX+lJSDeJBuv1MGFA4ITa4YaRywBBdZTcSl9cwLGB1X9ksTga5cd1XZeW/218w5cfmsgY6pgEsq/5MsjzbPqT20XDAhlC7GUmY3K+glAHfm8nVnihVdfmfU2apbx0bcDu5Er9mngykbj0u24P8VbkmlozS2H2dDmCjqQeE2Ux2pOuUxfqtMlJVSjAO8MCVvjc2nxsv55iHDTHGSlaN8TmsoBqVRbsOmauWFPQ7gFJDqF7YADS+20NDsgblAeeE23eZU4A/mG1dWfIK5E58nAoKWvnumFyQnJ0sMPpN" # File Paths -LOCAL_PATH = 'docs/priority_health/' # Replace with local +LOCAL_PATH = 'subset/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/integration_testing.ipynb b/src/integration_testing.ipynb new file mode 100644 index 0000000..e6dce29 --- /dev/null +++ b/src/integration_testing.ipynb @@ -0,0 +1,565 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import pandas as pd\n", + "import re\n", + "import ast\n", + "import json\n", + "\n", + "import prompt_funcs\n", + "import config\n", + "import utils\n", + "import preprocess\n", + "import table_funcs\n", + "import claude_funcs\n", + "import prompts" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "function clean_td(td):\n", + " Initialize td_clean as an empty list\n", + " for each dictionary d in td:\n", + " Initialize new_d as an empty dictionary\n", + " for each key k and value v in d:\n", + " if 'DATE' in k:\n", + " new_d[k] = v as a list\n", + " else if k is not 'page_num' or 'Filename':\n", + " if v is a string with commas:\n", + " new_d[k] = split v into a list by commas\n", + " else if v is 'N/A':\n", + " new_d[k] = an empty list\n", + " else:\n", + " new_d[k] = v as a list\n", + " else:\n", + " new_d[k] = v\n", + " Append new_d to td_clean\n", + " return td_clean\n", + "\n", + "function get_unique_keys(dicts):\n", + " Initialize keys as an empty set\n", + " for each dictionary d in dicts:\n", + " Add all keys in d to keys\n", + " return keys\n", + "\n", + "function get_unique_date_fields(td_results):\n", + " Initialize unique_date_fields as an empty dictionary\n", + " for each dictionary td in td_results:\n", + " for each key k and value v in td:\n", + " if 'DATE' is in k:\n", + " if k is not in unique_date_fields:\n", + " Initialize unique_date_fields[k] as an empty set\n", + " Add v to unique_date_fields[k] as a list\n", + " Convert sets in unique_date_fields to lists\n", + " return unique_date_fields\n", + "\n", + "function GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n", + " if field_name and values:\n", + " return a prompt string with page, field_name, and values\n", + " else:\n", + " return a prompt string with page, SERVICE, and METHODOLOGY from bu_dict\n", + "\n", + "function merge_results(td_results, bu_results, text_dict):\n", + " all_keys = get_unique_keys(td_results) union get_unique_keys(bu_results)\n", + " date_fields = get_unique_date_fields(td_results)\n", + " Initialize merged_results as an empty list\n", + " for each dictionary bu in bu_results:\n", + " Initialize merged as a dictionary with keys from all_keys and values from bu\n", + " td_on_page = filter td_results by page_num matching bu's page_num\n", + " for each dictionary td in td_on_page:\n", + " for each key k and value v in td:\n", + " if merged[k] is empty:\n", + " merged[k] = v\n", + " else if v is a list and merged[k] is not a list:\n", + " merged[k] = v\n", + " else if v is a list:\n", + " Extend merged[k] with v\n", + " for each date_key and date_values in date_fields:\n", + " if date_key is not in merged or merged[date_key] is empty:\n", + " merged[date_key] = date_values\n", + " else:\n", + " Extend merged[date_key] with values not already in merged[date_key]\n", + " for each key in all_keys:\n", + " if merged[key] is a list with more than one value:\n", + " page_text = get text for bu's page_num from text_dict\n", + " prompt = GENERATE_PROMPT(bu, td_on_page, page_text, key, merged[key])\n", + " response = invoke LLM with prompt\n", + " if response is valid:\n", + " merged[key] = response\n", + " else:\n", + " merged[key] = join merged[key] into a string\n", + " Append merged to merged_results\n", + " return merged_results\n", + "\n", + "function process_file(filename, input_dict):\n", + " contract_text = read contract text from input_dict using filename\n", + " Clean contract_text by removing newlines\n", + " Split contract_text into pages stored in text_dict\n", + " Run Top Down analysis and store results in td_results\n", + " Run Bottom Up analysis and store results in bu_results\n", + " Combine td_results and bu_results into combined_results\n", + " Create output directory based on filename\n", + " Save td_results, bu_results, and combined_results as CSV files in the output directory\n", + "\n", + "function process_all_files(input_dict):\n", + " for each filename in input_dict:\n", + " Call process_file with filename and input_dict\n", + " Print a message indicating the file has been processed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "\n", + "\n", + "\n", + "def clean_td(td):\n", + " td_clean = []\n", + " for d in td:\n", + " new_d = {}\n", + " for k, v in d.items():\n", + " if 'DATE' in k:\n", + " new_d[k] = v if isinstance(v, list) else [v]\n", + " elif k not in ['page_num', 'Filename']:\n", + " if isinstance(v, str) and ',' in v:\n", + " new_d[k] = [item.strip() for item in v.split(',')]\n", + " elif v == 'N/A':\n", + " new_d[k] = []\n", + " else:\n", + " new_d[k] = [v] if isinstance(v, str) else v\n", + " else:\n", + " new_d[k] = v\n", + " td_clean.append(new_d)\n", + " return td_clean\n", + "\n", + "def get_unique_keys(dicts):\n", + " keys = set()\n", + " for d in dicts:\n", + " keys.update(d.keys())\n", + " return keys\n", + "\n", + "def get_unique_date_fields(td_results):\n", + " unique_date_fields = {}\n", + " for td in td_results:\n", + " for key, value in td.items():\n", + " if 'DATE' in key:\n", + " if key not in unique_date_fields:\n", + " unique_date_fields[key] = set()\n", + " unique_date_fields[key].update(value if isinstance(value, list) else [value])\n", + " # Convert sets to lists for consistency\n", + " for key in unique_date_fields:\n", + " unique_date_fields[key] = list(unique_date_fields[key])\n", + " return unique_date_fields\n", + "\n", + "def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n", + " if field_name and values:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Field: {field_name}\n", + "Values: {values}\n", + "\n", + "Listed above is a page of a contract, as well as the potential values for the {field_name} field.\n", + "Your job is to identify which of these values is the most appropriate based on the page content.\n", + "\n", + "Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.\n", + "\"\"\"\n", + " else:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Service: {bu_dict['SERVICE']}\n", + "Methodology: {bu_dict['FULL_METHODOLOGY']}\n", + "\n", + "Listed above is a page of a contract, as well as a Service and Methodology found on this page. \n", + "Your job is to identify which of the following entities this specific Service is associated with. \n", + "\n", + "----Change Below----\n", + "\n", + "Here are the entities, represented as dictionary objects:\n", + "\n", + "{td_dicts}\n", + "\n", + "Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n", + "\"\"\"\n", + "\n", + "def merge_results(td_results, bu_results, text_dict):\n", + " all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results)\n", + " print(\"All Keys:\", all_keys)\n", + " \n", + " date_fields = get_unique_date_fields(td_results)\n", + " print(\"Date Fields:\", date_fields)\n", + " \n", + " merged_results = []\n", + " for bu in bu_results:\n", + " merged = {key: bu.get(key, \"\") for key in all_keys}\n", + " td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']]\n", + " \n", + " for td in td_on_page:\n", + " for key, value in td.items():\n", + " if not merged[key]:\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value and not isinstance(merged[key], list):\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value:\n", + " merged[key].extend(value)\n", + "\n", + " # Include unique date fields\n", + " for date_key, date_values in date_fields.items():\n", + " if date_key not in merged or not merged[date_key]:\n", + " merged[date_key] = date_values\n", + " else:\n", + " merged[date_key].extend([val for val in date_values if val not in merged[date_key]])\n", + "\n", + " # Use LLM for disambiguation and inference if necessary\n", + " for key in all_keys:\n", + " if isinstance(merged[key], list) and len(merged[key]) > 1:\n", + " page_num = bu['page_num']\n", + " page_text = text_dict.get(page_num, \"\")\n", + " prompt = GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key])\n", + " response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " try:\n", + " selected_value = response.strip()\n", + " merged[key] = selected_value\n", + " except Exception as e:\n", + " print(f\"Error processing LLM response for key {key}: {e}\")\n", + " merged[key] = ', '.join(merged[key])\n", + " \n", + " merged_results.append(merged)\n", + " \n", + " return merged_results\n", + "\n", + "def process_file(filename, input_dict):\n", + " contract_text = input_dict[filename]\n", + "\n", + " # Preprocess\n", + " contract_text = preprocess.clean_newlines(contract_text)\n", + " text_dict = preprocess.split_text(contract_text)\n", + " \n", + " # Run Top Down\n", + " td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page\n", + " print(\"TD Results:\", td_results)\n", + " \n", + " # Run Bottom Up\n", + " bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", + " print(\"BU Results:\", bu_results)\n", + " \n", + " # Combine\n", + " combined_results = merge_results(clean_td(td_results), bu_results, text_dict)\n", + " print(\"Combined Results:\", combined_results)\n", + " \n", + " # Create directories\n", + " base_filename = os.path.splitext(filename)[0]\n", + " output_dir = os.path.join('output', base_filename)\n", + " os.makedirs(output_dir, exist_ok=True)\n", + " \n", + " # Save results\n", + " pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n", + " pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n", + " pd.DataFrame(combined_results).to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", + " \n", + "def process_all_files(input_dict):\n", + " for filename in input_dict.keys():\n", + " process_file(filename, input_dict)\n", + " print(f\"Processed {filename}\")\n", + "\n", + "# Example usage\n", + "input_dict = utils.read_input(path='subset')\n", + "process_all_files(input_dict)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Preprocessing:\n", + "clean_newlines(contract_text):\n", + "\n", + "Removes unwanted newline characters from the contract text.\n", + "split_text(contract_text):\n", + "\n", + "Splits the contract text into individual pages and stores them in a dictionary.\n", + "Top Down and Bottom Up Analysis:\n", + "run_top_down(filename, text_dict):\n", + "\n", + "Extracts data from each page of the contract text and returns a list of dictionaries.\n", + "run_bottom_up(filename, text_dict):\n", + "\n", + "Extracts data from the entire contract document and returns a list of dictionaries.\n", + "Cleaning and Combining Results:\n", + "clean_td(td_results):\n", + "\n", + "Ensures values in the Top Down results are in list format and preserves date fields.\n", + "get_unique_keys(td_results, bu_results):\n", + "\n", + "Collects all unique field names from both Top Down and Bottom Up results.\n", + "get_unique_date_fields(td_results):\n", + "\n", + "Aggregates all unique date fields from the Top Down results.\n", + "merge_results(td_results, bu_results, text_dict):\n", + "\n", + "Combines Top Down and Bottom Up results, ensuring all relevant fields are included.\n", + "Uses the LLM to resolve ambiguities and infer the most appropriate values when necessary when multiple are there.\n", + "Post-Processing:\n", + "post_process_combined(combined_df):\n", + "\n", + "Cleans and validates specific fields (CONTRACT_LOB, PRODUCT, CONTRACT_NETWORK) in the combined dataset.\n", + "Adds new columns (Corrected_LOB, Corrected_PRODUCT, Corrected_NETWORK) to the combined dataset.\n", + "Extracts and standardizes page numbers.\n", + "clean_columns_combined(df, column_name, valid_values, new_column_name):\n", + "\n", + "Cleans a column by applying an exact match check and updates it to a new column.\n", + "clean_pagenumbers(df):\n", + "\n", + "Extracts and standardizes page numbers in the combined dataset.\n", + "Saving the Results:\n", + "process_file(filename, input_dict):\n", + "\n", + "Runs the entire process for a single file, from reading and preprocessing to saving the final results.\n", + "Calls the necessary functions to extract, combine, and post-process the data.\n", + "Saves the results as CSV files in the output directory.\n", + "process_all_files(input_dict):\n", + "\n", + "Iterates through all input files and processes each one using process_file." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "import postprocess\n", + "import utils\n", + "\n", + "def clean_td(td):\n", + " td_clean = []\n", + " for d in td:\n", + " new_d = {}\n", + " for k, v in d.items():\n", + " if 'DATE' in k:\n", + " new_d[k] = v if isinstance(v, list) else [v]\n", + " elif k not in ['page_num', 'Filename']:\n", + " if isinstance(v, str) and ',' in v:\n", + " new_d[k] = [item.strip() for item in v.split(',')]\n", + " elif v == 'N/A':\n", + " new_d[k] = []\n", + " else:\n", + " new_d[k] = [v] if isinstance(v, str) else v\n", + " else:\n", + " new_d[k] = v\n", + " td_clean.append(new_d)\n", + " return td_clean\n", + "\n", + "def get_unique_keys(dicts):\n", + " keys = set()\n", + " for d in dicts:\n", + " keys.update(d.keys())\n", + " return keys\n", + "\n", + "def get_unique_date_fields(td_results):\n", + " unique_date_fields = {}\n", + " for td in td_results:\n", + " for key, value in td.items():\n", + " if 'DATE' in key:\n", + " if key not in unique_date_fields:\n", + " unique_date_fields[key] = set()\n", + " unique_date_fields[key].update(value if isinstance(value, list) else [value])\n", + " for key in unique_date_fields:\n", + " unique_date_fields[key] = list(unique_date_fields[key])\n", + " return unique_date_fields\n", + "\n", + "def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n", + " if field_name and values:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Field: {field_name}\n", + "Values: {values}\n", + "\n", + "Listed above is a page of a contract, as well as the potential values for the {field_name} field.\n", + "Your job is to identify which of these values is the most appropriate based on the page content.\n", + "\n", + "Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.\n", + "\"\"\"\n", + " else:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Service: {bu_dict['SERVICE']}\n", + "Methodology: {bu_dict['FULL_METHODOLOGY']}\n", + "\n", + "Listed above is a page of a contract, as well as a Service and Methodology found on this page. \n", + "Your job is to identify which of the following entities this specific Service is associated with. \n", + "\n", + "----Change Below----\n", + "\n", + "Here are the entities, represented as dictionary objects:\n", + "\n", + "{td_dicts}\n", + "\n", + "Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n", + "\"\"\"\n", + "\n", + "def merge_results(td_results, bu_results, text_dict):\n", + " all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results)\n", + " print(\"All Keys:\", all_keys)\n", + " \n", + " date_fields = get_unique_date_fields(td_results)\n", + " print(\"Date Fields:\", date_fields)\n", + " \n", + " merged_results = []\n", + " for bu in bu_results:\n", + " merged = {key: bu.get(key, \"\") for key in all_keys}\n", + " td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']]\n", + " \n", + " for td in td_on_page:\n", + " for key, value in td.items():\n", + " if not merged[key]:\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value and not isinstance(merged[key], list):\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value:\n", + " merged[key].extend(value)\n", + "\n", + " for date_key, date_values in date_fields.items():\n", + " if date_key not in merged or not merged[date_key]:\n", + " merged[date_key] = date_values\n", + " else:\n", + " merged[date_key].extend([val for val in date_values if val not in merged[date_key]])\n", + "\n", + " for key in all_keys:\n", + " if isinstance(merged[key], list) and len(merged[key]) > 1:\n", + " page_num = bu['page_num']\n", + " page_text = text_dict.get(page_num, \"\")\n", + " prompt = GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key])\n", + " response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " try:\n", + " selected_value = response.strip()\n", + " merged[key] = selected_value\n", + " except Exception as e:\n", + " print(f\"Error processing LLM response for key {key}: {e}\")\n", + " merged[key] = ', '.join(merged[key])\n", + " \n", + " merged_results.append(merged)\n", + " \n", + " return merged_results\n", + "\n", + "def process_file(filename, input_dict):\n", + " contract_text = input_dict[filename]\n", + "\n", + " # Preprocess\n", + " contract_text = preprocess.clean_newlines(contract_text)\n", + " text_dict = preprocess.split_text(contract_text)\n", + " \n", + " # Run Top Down\n", + " td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page\n", + " print(\"TD Results:\", td_results)\n", + " \n", + " # Run Bottom Up\n", + " bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", + " print(\"BU Results:\", bu_results)\n", + " \n", + " # Combine\n", + " combined_results = merge_results(clean_td(td_results), bu_results, text_dict)\n", + " print(\"Combined Results:\", combined_results)\n", + " \n", + " # Convert to DataFrame\n", + " combined_df = pd.DataFrame(combined_results)\n", + " \n", + " # Post-process combined results\n", + " post_processed_combined_df = postprocess.postprocess_results(combined_df)\n", + " print(\"Post-processed Results:\", post_processed_combined_df)\n", + " \n", + " # Create directories\n", + " base_filename = os.path.splitext(filename)[0]\n", + " output_dir = os.path.join('output', base_filename)\n", + " os.makedirs(output_dir, exist_ok=True)\n", + " \n", + " # Save results\n", + " pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n", + " pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n", + " post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", + "\n", + "def process_all_files(input_dict):\n", + " for filename in input_dict.keys():\n", + " process_file(filename, input_dict)\n", + " print(f\"Processed {filename}\")\n", + "\n", + "# Example usage\n", + "input_dict = utils.read_input(path='subset')\n", + "process_all_files(input_dict)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "# Usage example:\n", + "#save_combined_to_csv(combined, 'combined_output.csv')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "[c['CONTRACT_LOB'] for c in combined]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/postprocess.py b/src/postprocess.py new file mode 100644 index 0000000..29d4149 --- /dev/null +++ b/src/postprocess.py @@ -0,0 +1,27 @@ +import pandas as pd +import postprocessingfuncs + +def sanitize_combined(df): + """ Sanitize the combined data by removing brackets and cleaning values. """ + for column in df.columns: + df[column] = df[column].apply(postprocessingfuncs.sanitize_value) + return df + +def post_process_combined(df): + VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] + VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] + VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] + + # Sanitize the combined data + df = sanitize_combined(df) + + postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') + postprocessingfuncs.clean_columns_combined(df, 'PRODUCT', VALID_PROGRAMS, 'Corrected_PRODUCT') + postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + + clean_df = postprocessingfuncs.clean_pagenumbers(df) + return clean_df + +def postprocess_results(combined_df): + combined_df = post_process_combined(combined_df) + return combined_df diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py new file mode 100644 index 0000000..9dfdd49 --- /dev/null +++ b/src/postprocessingfuncs.py @@ -0,0 +1,95 @@ +import pandas as pd +import re +import difflib + +def sanitize_value(value): + """ Remove brackets from list items and clean the values. """ + if pd.isna(value): + return value + if isinstance(value, str): + value = value.strip('[]') + if ',' in value: + value = ', '.join([item.strip(" '") for item in value.split(',')]) + else: + value = value.strip(" '") + return value + +def exact_match(val, valid_values): + val = val.strip().upper() + for valid_val in valid_values: + if val == valid_val.upper(): + return valid_val + return None + +def clean_columns_combined(df, column_name, valid_values, new_column_name): + """ Cleans a column by applying an exact match check and updates it to a new column. """ + changes = {} + original_values = df[column_name].unique() + + def update_column(entry): + if pd.notna(entry): + terms = sanitize_value(entry).split(',') + for term in terms: + match = exact_match(term, valid_values) + if match: + return match + return None + + df[new_column_name] = df[column_name].apply(update_column) + cleaned_values = df[new_column_name].unique() + return original_values, cleaned_values, changes + +def get_closest_match(val, valid_values, similarity_threshold=0.5): + if pd.isna(val): + return None + val = val.strip().upper() + matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold) + return matches[0] if matches else None + +def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): + """ Applies fuzzy matching to a column in the dataframe and logs changes. """ + changes = {} + original_values = df[column_name].unique() + + def log_and_clean(entry): + if pd.notna(entry): + words = sanitize_value(entry).split(',') + cleaned_words = [] + for word in words: + cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold) + if cleaned_word and word.strip().upper() != cleaned_word: + changes[word.strip()] = cleaned_word + cleaned_words.append(cleaned_word if cleaned_word else word.strip()) + return ', '.join(cleaned_words) + return None + + df[column_name] = df[column_name].apply(log_and_clean) + cleaned_values = df[column_name].unique() + return original_values, cleaned_values, changes + +def extract_page_number(page_text): + match = re.search(r'Pages\s+(\d+)-\d+', page_text) + if match: + return match.group(1) + else: + return page_text + +def clean_pagenumbers(df): + df['Page'] = df['Page'].apply(extract_page_number) + return df + +def consolidate_csv(df, output_csv='output/consolidated.csv'): + df['page_num'] = df['Page'].apply(lambda x: x.split()[1]) + + grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({ + 'PRODUCT': lambda x: ', '.join(x.dropna().unique()), + 'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()), + 'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()), + 'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()), + 'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()), + 'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()), + 'page_num' : lambda x : ', '.join(x.dropna().unique()) + }).reset_index() + + grouped.to_csv(output_csv, index=False) + print(f"Consolidated CSV has been saved to {output_csv}") diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index c7831b5..8811716 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -8,7 +8,7 @@ import json from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate from llama_index.core import SummaryIndex, Document from llama_index.llms.bedrock import Bedrock - +#pip install llama-index-llms-bedrock import config import prompts import claude_funcs @@ -227,6 +227,7 @@ def td_bu_combine(td, bu, text_dict): return combined_list # List of dictionaries + def run_prompts(file_object): filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") diff --git a/src/prompts.py b/src/prompts.py index a465112..9e8fa06 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -186,10 +186,38 @@ ONLY return the answer, with no other commentary or explanation. """ - - - - +def get_carveout_list(): + return ['Emergency Department/Emergency Room', 'Emergency Department', 'Emergency Room', 'Observation', 'Surgery', 'General Surgery', +'Ambulatory Surgery', 'Intensive Care', 'Trauma', 'HIV', 'Human Immunodeficiency Virus', 'Major Joint Replacement', 'Transplant', 'OBGYN', +'Obstetrician/Gynecologist', 'Obstetrician', 'Gynecologist', 'Opthalmology & Vision', 'Opthalmology', 'Vision', 'Never Events', +'Medically Unnecessary', 'Medically Unnecessary Procedures', 'Not Medically Necessary', 'Not Medically Necessary Procedures', +'Experimental', 'Investigational', 'Unlisted Codes', 'Durable Medical Equipment', 'DME', 'Prosthetics & Orthotics', 'Prosthetics', +'Orthotics', 'Implants', 'Hearing Aids', 'Hearing', 'Anesthesia', 'Anesthesiology', 'Medical Pharmacy', 'Physician Administered Drugs', +'Global', 'Bundled or Unbundled Codes', 'Bundled Codes', 'Unbundled Codes', 'Multiple Procedure Reductions', 'Second Surgery', 'Subsequent Surgeries', +'Non-Behavioral Health Mid-Level Professionals', 'Physician Assistant', 'PA', 'Nurse Practicioner', 'NP', 'Non-Physician', +'Non-Physician Health Professionals', 'Technical Component', 'Professional Component', 'Laboratory', 'Pathology', 'Lab', 'Path', 'Lab/Path', +'Laboratory/Pathology', 'Radiology', 'Imaging', 'Radiology/Imaging', 'Mammography', 'Diagnostic', 'Pre-Admission Procedures', +'Post-Discharge Procedures', 'Readmission', 'Status Indicators', 'Stop Loss', 'Emergency Medical', 'EMS', 'NICU', 'Neonatal Intensive Care Unit', +'Vaccine for Children', 'VFC', 'Surgical Assistant', 'Physicians/Clinical Psychologists', 'Doctor of Nursing Practice', 'Osteopathic Medicine', +'Clinical Psychology', 'Audiologist', 'Chiropractors', 'Registered Dietician', 'AUD', 'DC', 'RD', 'Board Certified Behavioral Analysis', 'Behavioral Analysis', +'BCBA', 'Independent Licensures', 'Licensed Professional Counselor', 'Marriage and Family Therapist', 'Substance Abuse Counselor', 'Clinical Social Worker', +'Behavioral Health Outpatient Clinic', 'Physical Therapist', 'Occupational Therapist', 'Speech Therapist', 'PT', 'OT', 'ST', 'Transportation', +'Primary Care', 'Primary Care Behavioral Health', 'Behavioral Physician', 'Clinical Psychologist', 'Mid-Level Practicioner', 'Dentist', 'Dental', +'Supplies and Devices', 'Supplies', 'Devices', 'Immunizations', 'Obstetrical Epidural', 'Pediatric Subspecialties', 'Orthopedic Surgery', 'All Other Specialists', +'Specialty Care Physician', 'Pediatric Primary Care', 'Nurse Anesthetist', 'Early Periodic Screening, Diagnostic, and Treatment', 'EPSDT', +'Ancillary', 'Oncology', 'Cancer', 'Inpatient Physical Rehabilitation', 'Inpatient Rehabilitation', 'Outpatient Rehabilitation', 'Rehabilitation', 'Non-Behavioral Health Rehabilitation', +'Extracorporeal Shock Wave Lithotripsy', 'Cardiac', 'Special Care Unit', 'Skilled Nursing', 'Infusion', 'Specialty Care', +'Specialty Care Physician', 'Organ Acquisition', 'Blood Products', 'Blood Products Outpatient', 'Blood Products Inpatient', 'High Cost Drugs', 'Sleep Studies', +'NICU', 'Newborn Intensive Care Unit', 'Extracorporeal Membrane Oxygenation', 'Burns', 'Kyphoplasty', 'Cryosurgical Ablation of the Prostate', +'Transurethral Thermal Ablation', 'TUMT', 'Transurethral Needle Ablation', 'TUNA', 'Hyperbaric Treatment', 'Clinic Visit', 'Boarder Baby', +'Pediatric Intensive Care Unit', 'PICU', 'Psychiatric', 'Mental Health', 'Behavioral Health', 'Substance Abuse', +'Behavioral Health and Substance Abuse', 'Sub-Acute Facility Care', 'Unrouped Inpatient', 'All Other Acute', +'Neurology', 'Neurology Subspecialties', 'Automatic Implantable Cardioverter Defibrillator', 'Percutaneous Transluminal Coronary Angioplasty', +'Non-Coronary Angioplasty', 'Cardiac Catheters', 'Cardiovascular Surgery', 'Cardiac Surgery', 'Cesarean Birth', 'Cesarean Section', 'C-Section', +'Gamma-Knife Radio-Surgery Outpatient', 'DaVinci Robotic Assisted Surgery', 'Outpatient Electrophysiology with Ablation', 'Outpatient Electrophysiology', +'Magnetic Resonance Image', 'MRI', 'Computed Tomography Scan', 'CT Scan', 'Radiation Therapy', 'Dialysis', 'Gastric Bypass', 'Lap Band', +'Obesity', 'Laparoscopic Cholecystectomy', 'Lap Chole', 'Laparoscopic Hysterectomy', 'Laparoscopic Prostatectomy', 'Laparoscopic Hysteroscopy', +'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology'] diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb deleted file mode 100644 index 9361dbd..0000000 --- a/src/test_notebook.ipynb +++ /dev/null @@ -1,359 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Imports\n", - "import pandas as pd\n", - "import re\n", - "import ast\n", - "import json\n", - "\n", - "import prompt_funcs\n", - "import config\n", - "import utils\n", - "import preprocess\n", - "import table_funcs\n", - "import claude_funcs\n", - "import prompts" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "input_dict = utils.read_input(path='../docs/test/')\n", - "input_dict.keys()\n", - "filename = '47-1928508 Advanced Anesthesia.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "contract_text = input_dict[filename]" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "#### PREPROCESS ####\n", - "contract_text = preprocess.clean_newlines(contract_text)\n", - "text_dict = preprocess.split_text(contract_text)\n", - "#text_dict = table_funcs.align_and_format_tables(text_dict)\n", - "#text_dict = preprocess.highlight_rates(text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "#### TOP DOWN ####\n", - "td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page\n", - "\n", - "#### BOTTOM UP ####\n", - "bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", - "\n", - "#### COMBINE ####\n", - "# combined_results = prompt_funcs.td_bu_combine(td_results, bu_results, text_dict)\n", - "\n", - "# answer_df = pd.DataFrame(combined_results)" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def clean_td(td):\n", - " td_clean = []\n", - " for d in td:\n", - " new_d = {}\n", - " for k in d.keys():\n", - " if k not in ['page_num', 'Filename']:\n", - " if ',' in d[k]:\n", - " if 'DATE' not in k:\n", - " new_d[k] = [v.strip() for v in d[k].split(',')]\n", - " elif d[k] == 'N/A':\n", - " new_d[k] = []\n", - " else:\n", - " new_d[k] = [d[k]]\n", - " else:\n", - " new_d[k] = d[k]\n", - " td_clean.append(new_d)\n", - " return td_clean\n", - "\n", - "def get_unique_td(td):\n", - " unique_td_dict = {}\n", - " for d in td:\n", - " for key, value in d.items():\n", - " if key not in ['page_num', 'Filename']:\n", - " for v in value:\n", - " if key not in unique_td_dict.keys():\n", - " unique_td_dict[key] = [v]\n", - " else:\n", - " unique_td_dict[key].append(v)\n", - " final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()}\n", - " return final_dict\n", - "\n", - "def td_bu_combine(td, bu, text_dict):\n", - " bu_dicts = bu.copy()\n", - " td_dicts = td.copy()\n", - "\n", - " td_dicts_clean = clean_td(td_dicts)\n", - " unique_td = get_unique_td(td_dicts_clean)\n", - " \n", - " combined_list = []\n", - " for bud_ in bu_dicts:\n", - " bud = bud_.copy()\n", - " for key, value in unique_td.items():\n", - " if len(value) == 0: # No value in contract\n", - " bud.update({key : \"\"})\n", - " elif len(value) == 1: # 1 value in contract\n", - " bud.update({key : value[0]})\n", - " else: # More than one value in contract\n", - " # Check for value on the page\n", - " page_value = []\n", - " for d in td_dicts_clean:\n", - " if (d['page_num'] == bud['page_num']) and (key in d.keys()):\n", - " for v in d[key]:\n", - " page_value.append(v)\n", - " \n", - " if len(page_value) == 1: # One value on page\n", - " bud.update({key : page_value[0]})\n", - " elif len(page_value) > 1: # Multiple values on page\n", - " bud.update({key : ', '.join(page_value)})\n", - " elif len(page_value) == 0:\n", - " bud.update({key : \"\"}) \n", - " combined_list.append(bud)\n", - " return combined_list" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "combined = td_bu_combine(td_results, bu_results, text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Medicaid',\n", - " 'Medicaid',\n", - " 'Medicaid',\n", - " 'Medicaid',\n", - " 'Medicaid',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '']" - ] - }, - "execution_count": 95, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "[c['CONTRACT_LOB'] for c in combined]" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [], - "source": [ - "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}