Table formatting for code improvement

src/bottom_up_funcs.py

@@ -8,6 +8,7 @@ import claude_funcs
 import preprocess
 import dict_operations
 import postprocess
+import table_funcs
 
 
 def run_bottom_up_lesser(d, text_dict, tokens=8000):
@@ -44,37 +45,45 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens):
     temp_dicts = []
     for d in answer_dicts:     
         # Bottom Up Lesser
-        lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens)
-        temp_dicts.append(lesser_object[1]) # Add original object from d
-        if lesser_object[0]: temp_dicts.append(lesser_object[0]) # If lesser, add lesser object
+        if config.RUN_LESSER:
+            lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens)
+            temp_dicts.append(lesser_object[1]) # Add original object from d
+            if lesser_object[0]: temp_dicts.append(lesser_object[0]) # If lesser, add lesser object
+        else:
+            temp_dicts.append(d)
     
     # Add additional fields to each row
     final_dicts = []
     for d in temp_dicts:
         if d is not None:
             page_num = d['page_num']
+
             # Bottom Up Methodology
-            prompt = prompts.BOTTOM_UP_METHODOLOGY(d)
-            methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
-            d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer
+            if config.RUN_METHODOLOGY:
+                prompt = prompts.BOTTOM_UP_METHODOLOGY(d)
+                methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
+                d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer
 
             # # Bottom Up FS
-            prompt = prompts.BOTTOM_UP_FS(d)
-            fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens)
-            fs_dict = dict_operations.secondary_string_to_dict(fs_answer)
-            d.update(fs_dict)
+            if config.RUN_FS:
+                prompt = prompts.BOTTOM_UP_FS(d)
+                fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens)
+                fs_dict = dict_operations.secondary_string_to_dict(fs_answer)
+                d.update(fs_dict)
 
             # # Bottom Up Exception/Escalator
-            prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[page_num])
-            exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens)
-            exc_dict = dict_operations.secondary_string_to_dict(exc_answer)
-            d.update(exc_dict)
+            if config.RUN_EXCEPTION:
+                prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[page_num])
+                exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens)
+                exc_dict = dict_operations.secondary_string_to_dict(exc_answer)
+                d.update(exc_dict)
 
             # # Bottom Up Codes
-            prompt = prompts.BOTTOM_UP_CODES(d, text_dict[page_num])
-            codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens)
-            codes_dict = dict_operations.secondary_string_to_dict(codes_answer)
-            d.update(codes_dict)
+            if config.RUN_CODES:
+                prompt = prompts.BOTTOM_UP_CODES(d, text_dict[page_num])
+                codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens)
+                codes_dict = dict_operations.secondary_string_to_dict(codes_answer)
+                d.update(codes_dict)
 
             final_dicts.append(d)
         
@@ -103,14 +112,15 @@ def consolidate_lob(lob_dicts, results_dicts):
             final_dicts.append(d)
     return final_dicts
 
+
 def bottom_up(file_object):
-    #### PREPROCESS ####
     filename, contract_text = file_object
     if config.VERBOSE: print(f"Processing {filename}...")
 
+    #### PREPROCESS ####
     contract_text = preprocess.clean_newlines(contract_text)
     text_dict = preprocess.split_text(contract_text)
-    text_dict = preprocess.align_tables(text_dict)
+    text_dict = table_funcs.align_and_format_tables(text_dict)
     text_dict = preprocess.highlight_rates(text_dict)
 
     #### PROMPT ####
@@ -120,22 +130,27 @@ def bottom_up(file_object):
     answer_dicts = postprocess.primary_filter(answer_dicts)
 
     # Bottom Up LOB - CONTRACT_LOB, CONTRACT_PROGRAM, CONTRACT_PRODUCT, PROV_TYPE
-    lob_dicts = run_bottom_up_lob(answer_dicts, text_dict, 4000)
-
+    if config.RUN_LOB:
+        lob_dicts = run_bottom_up_lob(answer_dicts, text_dict, 4000)
+    
     # Bottom Up Secondary
     results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000)
 
     # Append LOB
-    final_dicts = consolidate_lob(lob_dicts, results_dicts)
-
-    #### POSTPROCESS ####
-    #answer_dicts = append_secondary(answer_dicts)
+    if config.RUN_LOB:
+        final_dicts = consolidate_lob(lob_dicts, results_dicts)
+    else:
+        final_dicts = results_dicts
 
     for d in final_dicts:
         d['Filename'] = filename
 
     answer_df = pd.DataFrame(final_dicts)
 
+    #### POSTPROCESS ####
+    #answer_dicts = append_secondary(answer_dicts)
+    answer_df = postprocess.bottom_up_postprocess(answer_df)
+
     # Write output
     if config.WRITE_OUTPUT:
         if config.OUTPUT_MODE == '_INDIVIDUAL_': 

src/config.py

@@ -16,10 +16,22 @@ OUTPUT_MODE = '_INDIVIDUAL_'  # or'_CONSOLIDATED_'
 # Multithread Settings
 MAX_WORKERS = 20
 
+# Prompt Debugging
+RUN_PRIMARY = True # Always True
+RUN_LOB = True
+RUN_LESSER = True
+RUN_METHODOLOGY = True
+RUN_FS = True
+RUN_EXCEPTION = True
+RUN_CODES = True
+
 # AWS Keys
+AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAOY7QOHS"
+AWS_SECRET_ACCESS_KEY="U3zdt9cAgCFVdy48uwktZSx2owC4QXo4/mtWCwdQ"
+AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIv//////////wEaCXVzLWVhc3QtMiJHMEUCIQCxyHC2X3koEVwj1v5GQCySf/crrB3a1muqvXsMEUOvjwIgbX9sZ25cw8Hc0zklioXs/5rq9hfXna4AeAynR3O0+eUqjAMI5P//////////ARAAGgw2NjAxMzEwNjg3ODIiDDZ45+avGABDdEaUiirgAkJzeXOOSyKxFQv6Mq3sCSFlAGkjzUuMAziu1RZdWcF99u0rQthauWdxJ6rU4kwfd/M2LoWRQgu/8VxsktWxcivbleRrxwGAtcayMOBDWzeCdCDnvWZgKxZULiefEeJK3BDwUsWRN541FgZ/xqsAoASQlMO98rpNZ6W8xNq961xSpVUItMRPjOuHAot9YhPi1xLl+haeIdpsxGhx4ztQuJ+/ag5KAVrKpEiSsrQqf2L/q4FHB/3/YYzzAgXqP7Wy4qoPsWxfARvCw1FAoOqqPGjcmFwD3kov8q0FyYG4cc7/0UySSHxl7qUHFAr8vPuF7i+BdAFxeZEpAFd4s00gIm0V4ne8ta0wELiIfbkULdKZu7UezROqhm4+q9nQByrog4d1+YRE7B6x3zA9koFogcHQizZtZyoBDYK6Q3p+eab+SjIW6NndyYhCT1diebVLfnpth0MDMnIHibMHetm+XGwwiMnrsQY6pgGKQdkRrc2lhFuoRtw4uh4GTTz9PTQap3Ts2UNe7sT/49TpWxtqSB5WUyFGn2GAqnh1sOBZON3jo1e8qjibVxfACfv38PUBlktj6bVI0izhKfcEdoSyYWW1Cs5CTt1wlF3sOu5ztkTB+Gq8+H20gYAH+BgctDy43H6OwCGbBcveRxOEVH9jn+9GC6I99GNmjQ0w3Is9jT7zaJf52lE9nAA4n/p64Rro"
 
 # File Paths
-LOCAL_PATH = 'docs/test/' # Replace with local
+LOCAL_PATH = '../docs/test/' # Replace with local
 
 # S3 Settings
 S3_CLIENT = boto3.client('s3',

src/postprocess.py

@@ -10,3 +10,17 @@ def primary_filter(answer_dict):
         else:
             filtered_dicts.append(d)
     return filtered_dicts
+
+
+def bottom_up_postprocess(df):
+    # Remove Unnamed columns
+    df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]]
+
+    # Reimbursement Flat Fee
+    df.loc[:, 'REIMBURSEMENT_FLAT_FEE'] = df['REIMBURSEMENT_FLAT_FEE'].astype(str).str.replace(r'[^\d.]', '', regex=True)
+
+    # Reimbursement Rate
+    df.loc[:, 'REIMBURSEMENT_RATE'] = df['REIMBURSEMENT_RATE'].astype(str).str.replace(r'%|<<<|>>>|[^\d.]', '', regex=True)
+
+    return df
+

src/preprocess.py

@@ -10,19 +10,6 @@ def split_text(text):
     text_dict = {page.split()[0] : page for page in text_list}
     return text_dict
 
-def align_tables(text_dict):
-    pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL)
-    for page, text in text_dict.items():
-        tables = pattern.findall(text)
-        for table in tables:
-            pre_table = table.split('{')[0].strip() # Get the pretable text for alignment
-            text = text.replace(table, '') # Remove full table
-            text = text.replace(pre_table, table) # replace remaining pretable text with full table
-            text = text.replace('-------Table Start--------', '')
-            text = text.replace('-------Table End--------', '')   
-        text_dict[page] = text
-    return text_dict
-
 def highlight_rates(text_dict):
     for page, text in text_dict.items():
         words = text.split() 

src/table_funcs.py

@@ -17,4 +17,45 @@ def extract_all_tables(input_dict):
     tables_dict = {}
     for filename, filetext in input_dict.items():
         tables_dict[filename] = extract_tables(filetext)
-    return tables_dict
+    return tables_dict
+
+def format_table(table):
+    table = table.replace('>>>', '').replace('<<<', '')
+    table_dict = eval(table)
+    
+    keys = list(table_dict.keys())
+    # If values are lists
+    if isinstance(table_dict[keys[0]], list):
+        final_string = ""
+        for i in range(len(table_dict[keys[0]])):
+            for k in keys:
+                key_string = k + ': ' + table_dict[k][i] + ', '
+                final_string += key_string
+            final_string += '|\n'
+    else:
+    # If values are not lists
+        final_string = ""
+        for k in keys:
+            key_string = k + ': ' + table_dict[k] + ', '
+            final_string += key_string
+        final_string += '|\n'
+    return final_string
+
+def align_and_format_tables(text_dict):
+    pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL)
+    for page, text in text_dict.items():
+        tables = pattern.findall(text)
+        for table in tables:
+            pre_table = table.split('{')[0].strip() # Get the pretable text for alignment
+            text = text.replace(table, '') # Remove full table
+            table_only = table.replace(pre_table, '')
+            try:
+                formatted_table = format_table(table_only)
+            except:
+                formatted_table = table
+            text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table
+            text = text.replace('-------Table Start--------', '')
+            text = text.replace('-------Table End--------', '')   
+        text_dict[page] = text
+    return text_dict
+