Updated exception/escalator

src/config.py

@@ -54,9 +54,9 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R
 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK']
 
 # AWS Keys
-AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMRIB6IY4"
-AWS_SECRET_ACCESS_KEY="o+w4z8WyJeQi+l9O9LKAuWNpPogXAJZoS1VSgUJv"
-AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGAaCXVzLWVhc3QtMiJHMEUCIQCt/8UZmc37xOqSNWy1mhtFzBRR4aJ2jjSI+DZXsL+dgQIgIKSurMIGok1CIpwkWzZetBEFZJuFrChMsnPq1GMl6tEqjAMI+f//////////ARAAGgw2NjAxMzEwNjg3ODIiDG/YpM3sZ+xicr8GcCrgAp5GZeUYm818DYciItJblOTIk0oNaPR73NMXCXHHeHORGR1KJiXTLinqR2OpI/GGX/AsSE6yIpuu2oEQXPUyM0RN9wSpPyVSWDtHmrtj19qsDMTDwLQeTQITBEEKpohhpWNX6y9kSgDF/2QOXxq1uI7mZlXBobdp0XfN7mEvQWBz6UH5EKvc3clkCWiE3GhlhvNsBFnIZSklPyy8jpkxQ1SL0xDeNUWHO9YzWlkrg3dM5T3ZUXk1IW7YiVEUQs/kbG6rVpMSSuINApd3AUNnMY/hBvc32CBOyK/9VgMMI+5hvqd+nUEDUE2zNa/e9YBHqn8JbsQG4cDIQasdD2G1CgDYLvGwJt2x3l+O7w61yfvmknoVOYROKDA2mmA9/XWvhPLUjTDR8Dl3iIcW8YIROVNb9GHMM/t1CX6p5VtiLYbKhPqn7YS44ulnbqbKdRW1MBKTmzzeLQquxZK19JZtBd0wnpzDswY6pgFSmdwIcQBYWdVBtaQNycF5p8041L0kZ4mLe3oqa4DQCHwZMeLA5LZP4c/VSMVHCnNM2rovqyO2A12pLgWvlsXIzitIuL+r405Sk/tJT3BXDuyxKGIS6wvP6eLteSrCCCyNXT+qipoRugVStkObsvBBGT/I535cDlb2DiG6eoPBkwnPvcgJbEJMK2UqNoTJAiuQhP1cmFIV5OratlRC7GjPqnmNxZOQ"
+AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFY5DMLIK"
+AWS_SECRET_ACCESS_KEY="IN2w1JrXU4f7rKR2IbiOwCZNUrgls3ZK/65w8cCS"
+AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHMaCXVzLWVhc3QtMiJHMEUCIEhqvUNWd6r+bPwIIFkg242wxJGTn9k7qWiWVZSXn4dLAiEAr01immSRrcDW1oqW0QfN3gqivPNQV8JkTgLipIW0rpsqgwMIHBAAGgw2NjAxMzEwNjg3ODIiDJCiH+aROTYN8KdY2CrgAmCbjJwVL63njz5qxkOtexudA6CZ7Kz5BGpaA2WbdyHHc8j/UB/44MfOAxrTeLiP6w12qpYMc7U5GdZIULvNwWxJhJmnOHwLmLU0XuOtc52l3VCQmXLLMj1Q/dVPm+MVFgTM4RLTTNRGRE9boX1MP92pZlUZ/sjSKBl2ELfiJkrgzOkS3kYADmICJ0wWU1Ndqpi8sGTs8JHCyHCj3TSkJVaeCcwyvWssZHrgOv7/MUdgeHkiJx3m/MJNdOmxyZzr7mWtAFgJMODr8zM/vAPWogzcD0MEoMpa98ag9fcC6nE9us4KpiIIGAJNzXX6YpjkxoU+/tdXwMLsbkqmnqblz3zvB6jEyqTLMeEMYfexlzA7bBrq6Ooh2+nvTpFwF3E4RTdXsOdjgiw2nTwfjBYFT4cAGz9rQl2hZCLdhEhwZu5dmF/oKn6Nd5eec+KV0+Ep/SPor9p/uQh597zDDWt1BMwwurvHswY6pgFAChD2esP7NSzOQ+dIKA0CTZuNwdK7mjfkjtGEPsjDJBF3ueD4CybkHrqM2Ud6xhI8LwZU6DiIdUeqFXQHawdF1wDegBgy1Tymv1nCOZYCyTbW6ihJ+L4oJsr8XJwKpUhRgQQ8ECscAWFGRMN3zPt6T+OAcxJvjnKtdQO+vOCJNMzkr1yb7cuEoUcWPSwFrVkZg5iKP4T9DLLDCfGZUiV7AND6trJ+"
 
 # File Paths
 LOCAL_PATH = 'data/test/' # Replace with local

src/postprocessingfuncs.py

@@ -227,7 +227,7 @@ def filter_service_column(df):
     df = df.dropna(subset=['SERVICE'])
 
     keywords = [
-        'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST'
+        'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program'
     ]
     pattern = '|'.join(keywords)
     df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)]

src/preprocess.py

@@ -79,7 +79,7 @@ def chunk_text(text_dict):
     Returns:
         dict: A new dictionary where each key is a string representing the range of pages in a chunk, and each value is the concatenated text of those pages.
     """
-    special_pages = {int(page): text for page, text in text_dict.items() if '%' in text or '$' in text}
+    special_pages = {int(page): text for page, text in text_dict.items() if (('%' in text) or ('$' in text) or ('percent' in text))}
     page_numbers = sorted(special_pages.keys())
     chunk_page_numbers = []
     for page in page_numbers:

src/prompt_funcs.py

@@ -61,7 +61,7 @@ def run_bottom_up_primary(text_dict, tokens):
         #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]:
             #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME)
     for page_number in text_dict.keys():
-        if  page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number]):
+        if  page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number] or 'percent' in text_dict[page_number].lower()):
             # Run Primary
             prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME)
             answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens)

src/prompts.py

@@ -38,7 +38,7 @@ def BOTTOM_UP_PRIMARY(page, payer):
 
 The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. 
 
-The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections identified as examples - these must be omitted from output.
+The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections explicitly identified as examples - these must be omitted from output.
 
 If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.
 
@@ -48,7 +48,7 @@ Here are the attributes to be included in each dictionary, and instructions on h
 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A.
 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
-'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. 
+'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology. 
 
 Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary.
 
@@ -94,7 +94,9 @@ Your job is to identify if the listed reimbursement contains some sort of except
 Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided):
 REIMBURSEMENT_EXCEPTION_IND: If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, 'N'
 REIMBURSEMENT_DESCRIBE_EXCEPTION: If the listed reimbursement contains some sort of exception, describe it here. Try to use exact words from the text only. If there is none, just write 'N/A'
-RATE_ESCALATOR_IND: If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'.
+RATE_ESCALATOR_IND: Does the listed reimbursement include annual rate escalators for year-over-year rate increases? If yes, return 'Y'. Otherwise, 'N'.
+RATE_ESCALATOR_BASIS: If contract does include annual rate escalators, on what are they based? Do not be overly verbose in your answer.
+RATE_ESCALATOR_YEARLY_PERCENT_INCREASE: What is the percent value of increase or decrease year-over-year? If a decrease, return a negative percentage.
 
 Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
 """