jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

reformatting

Browse Source
This commit is contained in:
Michael McGuinness
2024-09-30 15:28:07 +01:00
parent 7285f56d79
commit 7772d1833a
12 changed files with 232 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/bottom_up_funcs.py
+4 -5
View File
@@ -28,7 +28,7 @@ def run_bottom_up(filename, text_dict):
answer_strings = run_bottom_up_primary(text_dict, 8000)
answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename)
answer_dicts_filtered = postprocess_funcs.filter_service_column(answer_dicts)
# Bottom Up Secondary
results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000)
@@ -37,9 +37,9 @@ def run_bottom_up(filename, text_dict):
# Add Filename
for d in results_dicts:
d['Filename'] = filename
return results_dicts # List of dictionaries
d["Filename"] = filename
return results_dicts # List of dictionaries
def run_bottom_up_primary(text_dict, tokens):
@@ -147,7 +147,6 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens):
return final_dicts
def run_bottom_up_lesser(d, text_dict, tokens=4000):
"""
Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of',
fieldExtraction/src/categorization.py
+20 -9
View File
@@ -1,4 +1,3 @@
import os
import utils
@@ -7,28 +6,40 @@ import table_funcs
def contains_reimbursement(text_dict, page):
return ('%' in text_dict[page] or '$' in text_dict[page] or 'percent' in text_dict[page].lower())
return (
"%" in text_dict[page]
or "$" in text_dict[page]
or "percent" in text_dict[page].lower()
)
directory_path = "C:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\data\\centene_healthnet"
#input_dict = utils.read_input(path=folder_path)
directory_path = (
"C:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\data\\centene_healthnet"
)
# input_dict = utils.read_input(path=folder_path)
all_files = {}
reimbursement_count = 0
total_count = 0
# Walk through all directories and files in the directory
for root, dirs, files in os.walk(directory_path):
for file in files:
if file.endswith('.txt'):
if file.endswith(".txt"):
# Construct full file path
file_path = os.path.join(root, file)
print(file_path)
try:
with open(file_path, 'r', encoding='utf-8') as file:
with open(file_path, "r", encoding="utf-8") as file:
contract_text = file.read()
if ('%' in contract_text or '$' in contract_text or 'percent' in contract_text.lower()):
if (
"%" in contract_text
or "$" in contract_text
or "percent" in contract_text.lower()
):
reimbursement_count += 1
total_count += 1
except:
continue
print(f"For Centene HealthNet, {reimbursement_count}/{total_count} files contain potential reimbursement terms.")
print(
f"For Centene HealthNet, {reimbursement_count}/{total_count} files contain potential reimbursement terms."
)
fieldExtraction/src/config.py
+19 -18
View File
@@ -70,22 +70,23 @@ CLIENT = os.getenv("CLIENT", "base")
VALID_COLUMNS = columns.VALID_COLUMNS_TEMPLATE[CLIENT]
# AWS Keys
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
AWS_SESSION_TOKEN = os.getenv('AWS_SESSION_TOKEN')
AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN")
# File Paths
LOCAL_PATH = os.getenv('LOCAL_PATH', "data/")
LOCAL_PATH = os.getenv("LOCAL_PATH", "data/")
# S3 Settings
S3_CLIENT = boto3.client('s3',
region_name="us-east-2",
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_session_token=AWS_SESSION_TOKEN
)
BUCKET = os.getenv('BUCKET', "doczy-dev-infra-textract")
PREFIX = os.getenv('BUCKET_PREFIX')
S3_CLIENT = boto3.client(
"s3",
region_name="us-east-2",
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_session_token=AWS_SESSION_TOKEN,
)
BUCKET = os.getenv("BUCKET", "doczy-dev-infra-textract")
PREFIX = os.getenv("BUCKET_PREFIX")
# Bedrock Settings
BEDROCK_RUNTIME = boto3.client(
@@ -101,9 +102,9 @@ MODEL_ID_CLAUDE2 = "anthropic.claude-instant-v1"
# LLama Model
# LLAMA_MODEL = Bedrock(
# model=MODEL_ID_CLAUDE3_HAIKU,
# aws_access_key_id=AWS_ACCESS_KEY_ID,
# aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
# aws_session_token=AWS_SESSION_TOKEN,
# region_name="us-east-1"
# )
# model=MODEL_ID_CLAUDE3_HAIKU,
# aws_access_key_id=AWS_ACCESS_KEY_ID,
# aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
# aws_session_token=AWS_SESSION_TOKEN,
# region_name="us-east-1"
# )
fieldExtraction/src/file_processing.py
+8 -5
View File
@@ -49,20 +49,23 @@ def process_file(file_object):
text_dict = preprocess.highlight_rates(text_dict)
################## RUN TOP DOWN ##################
td_results = top_down_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page
td_results = top_down_funcs.run_top_down(
filename, text_dict
) # Returns list of dictionaries for each page
print(f"Top Down Complete - {filename}")
################## RUN BOTTOM UP ##################
bu_results = bottom_up_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries
bu_results = bottom_up_funcs.run_bottom_up(
filename, text_dict
) # Returns list of dictionaries
print(f"Bottom Up Complete - {filename}")
################## COMBINE TOP DOWN AND BOTTOM UP ##################
combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict)
print(f"TD/BU Merge Complete - {filename} ")
################## RUN POSTPROCESSING ##################
combined_df = pd.DataFrame(combined_results)
post_processed_combined_df = postprocess.postprocess_results(combined_df)
return post_processed_combined_df
fieldExtraction/src/main.py
+15 -6
View File
@@ -62,14 +62,23 @@ def main():
# Write Consolidated Output
if config.WRITE_OUTPUT:
all_results_df = pd.concat(all_results).reset_index(drop=True)
all_results_df.drop('page_num', axis=1, inplace=True)
# Reorder columns
final_df = all_results_df[[col for col in config.VALID_COLUMNS if col in all_results_df.columns] + [col for col in all_results_df.columns if col not in config.VALID_COLUMNS]]
final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH))
all_results_df.drop("page_num", axis=1, inplace=True)
# Reorder columns
final_df = all_results_df[
[col for col in config.VALID_COLUMNS if col in all_results_df.columns]
+ [
col
for col in all_results_df.columns
if col not in config.VALID_COLUMNS
]
]
final_df.to_csv(
os.path.join(
config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH
)
)
if __name__ == "__main__":
main()
fieldExtraction/src/merge_funcs.py
+17 -8
View File
@@ -7,6 +7,7 @@ def prompt_select_multiple(key, value, bu_dict, page):
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
return answer
def merge_results(td_results, bu_results, text_dict):
def find_td_dict(page_num):
"""Helper function to find dictionary in td_results with matching page_num"""
@@ -20,20 +21,26 @@ def merge_results(td_results, bu_results, text_dict):
if td_dict is None:
# If there's no matching page, use recursion to look one page earlier
if int(page_num) > 1:
return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict), str(int(page_num) - 1)
return get_value(
find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict
), str(int(page_num) - 1)
else:
return "N/A", "N/A" # Base case if no previous pages exist
else:
value = td_dict.get(key, "N/A")
if value == "N/A" or value == "":
# If value is 'N/A' or empty, use recursion to look one page earlier
return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict), str(int(page_num) - 1)
elif ',' in value and 'DATE' not in key:
return get_value(
find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict
), str(int(page_num) - 1)
elif "," in value and "DATE" not in key:
# Randomly select one of the comma-separated values
return prompt_select_multiple(key, value, bu_dict, text_dict[page_num]), str(int(page_num))
return prompt_select_multiple(
key, value, bu_dict, text_dict[page_num]
), str(int(page_num))
else:
return value, page_num
merged_results = []
for bu_dict in bu_results:
page_num = bu_dict["page_num"]
@@ -43,8 +50,10 @@ def merge_results(td_results, bu_results, text_dict):
# Add or overwrite keys from td_dict
if td_dict:
for key, value in td_dict.items():
if key not in ['Filename', 'page_num']:
new_dict[key], new_dict[f"{key}_PG"] = get_value(td_dict, key, page_num, text_dict)
if key not in ["Filename", "page_num"]:
new_dict[key], new_dict[f"{key}_PG"] = get_value(
td_dict, key, page_num, text_dict
)
merged_results.append(new_dict)
return merged_results
fieldExtraction/src/postprocess.py
+12 -4
View File
@@ -27,18 +27,26 @@ def postprocess_results(combined_df):
["CONTRACT_NETWORK", "Corrected_NETWORK", config.VALID_NETWORKS],
]:
try:
postprocess_funcs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1])
postprocess_funcs.clean_columns_combined(
df, field_list[0], field_list[2], field_list[1]
)
except Exception as e:
print(f"Postprocessing Error - clean_columns_combined : {e}")
try:
postprocess_funcs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD)
postprocess_funcs.clean_columns_combined_fuzzy(
df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD
)
except Exception as e:
print(f"Postprocessing Error - clean_columns_combined_fuzzy : {e}")
# Correct misplaced values across columns
try:
df = postprocess_funcs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict)
df = postprocess_funcs.correct_misplaced_values(
df,
["CONTRACT_LOB", "CONTRACT_PROGRAM", "CONTRACT_NETWORK"],
valid_values_dict,
)
except Exception as e:
print(f"Postprocessing Error - correct_misplaced_values : {e}")
@@ -73,4 +81,4 @@ def postprocess_results(combined_df):
]
df = df[column_order]
return df
return df
fieldExtraction/src/postprocess_funcs.py
+119 -57
View File
@@ -1,10 +1,10 @@
import pandas as pd
import re
import difflib
import config
def sanitize_value(value):
"""
Sanitizes a given value by handling nulls, lists, and strings to ensure consistency in formatting.
@@ -22,15 +22,16 @@ def sanitize_value(value):
"""
try:
if isinstance(value, list):
return ', '.join(str(v) for v in value)
return ", ".join(str(v) for v in value)
elif isinstance(value, str):
value = value.strip('[]')
return ', '.join([item.strip(" '") for item in value.split(',')])
value = value.strip("[]")
return ", ".join([item.strip(" '") for item in value.split(",")])
elif pd.isna(value):
return "N/A"
except:
return value
def exact_match(val, valid_values):
"""
Compares a given string value against a list of valid values to determine if there is an exact match, case-insensitively.
@@ -52,6 +53,7 @@ def exact_match(val, valid_values):
return valid_val
return None
def clean_columns_combined(df, column_name, valid_values, new_column_name):
"""
Cleans and standardizes entries in a specified column of a DataFrame based on a list of valid values, creating a new column with standardized values.
@@ -75,13 +77,13 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name):
def update_column(entry):
if pd.notna(entry):
terms = entry.split(',')
terms = entry.split(",")
for term in terms:
match = exact_match(term, valid_values)
if match:
return match
return None
df[new_column_name] = df[column_name].apply(update_column)
cleaned_values = df[new_column_name].unique()
return original_values, cleaned_values, changes
@@ -92,7 +94,7 @@ def get_closest_match(val, valid_values, similarity_threshold=0.7):
Finds the closest match for a given string from a list of valid values, using a similarity threshold.
This function cleans the input value and compares it to each cleaned value in the valid values list using a fuzzy
matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no
matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no
matches meet the threshold, the function returns None.
Parameters:
@@ -106,7 +108,9 @@ def get_closest_match(val, valid_values, similarity_threshold=0.7):
if pd.isna(val):
return None
val = val.strip().upper()
matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold)
matches = difflib.get_close_matches(
val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold
)
return matches[0] if matches else None
@@ -131,17 +135,19 @@ def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
changes = {}
df[column_name] = df[column_name].apply(sanitize_value)
original_values = df[column_name].unique()
def log_and_clean(entry):
if pd.notna(entry):
words = entry.split(',')
words = entry.split(",")
cleaned_words = []
for word in words:
cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
cleaned_word = get_closest_match(
word.strip(), valid_values, similarity_threshold=threshold
)
if cleaned_word and word.strip().upper() != cleaned_word:
changes[word.strip()] = cleaned_word
cleaned_words.append(cleaned_word if cleaned_word else word.strip())
return ', '.join(cleaned_words)
return ", ".join(cleaned_words)
return None
df[column_name] = df[column_name].apply(log_and_clean)
@@ -149,7 +155,6 @@ def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
return original_values, cleaned_values, changes
def correct_misplaced_values(df, columns, valid_values_dict):
"""
Corrects misplaced values within specified columns of a DataFrame based on a dictionary of valid values for each column.
@@ -170,20 +175,32 @@ def correct_misplaced_values(df, columns, valid_values_dict):
for index, row in df.iterrows():
for col in columns:
if pd.notna(row[col]):
terms = row[col].split(',')
terms = row[col].split(",")
for term in terms:
term = term.strip()
for target_col, valid_values in valid_values_dict.items():
if target_col != col:
match = exact_match(term, valid_values)
if match:
if pd.isna(row[target_col]) or not row[target_col].strip():
if (
pd.isna(row[target_col])
or not row[target_col].strip()
):
df.at[index, target_col] = match
df.at[index, col] = None
else:
current_value = row[target_col].strip()
if get_closest_match(match, [current_value], config.FUZZY_MATCH_THRESHOLD) is None:
df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell"
if (
get_closest_match(
match,
[current_value],
config.FUZZY_MATCH_THRESHOLD,
)
is None
):
df.at[index, "Corrected_" + target_col] = (
f"Found {term} in {col} cell"
)
df.at[index, col] = None
return df
@@ -204,13 +221,28 @@ def filter_service_column(d):
list: A new list of dictionaries with items containing specified keywords in the 'SERVICE' key removed.
"""
keywords = [
'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED',
'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES',
"LIABILITY",
"RISK",
"LOBBYING",
"DAMAGES",
"CONFIDENTIALITY",
"ARBITRATION",
"FALSE CLAIMS ACT",
"UNSPECIFIED",
"AUDIT",
"INTEREST",
"N/A",
"BUSINESS",
"COMPLIANCE",
"Medical Assistance Program",
"MATERIAL SUBCONTRACT",
"GIFTS",
"GRATUITIES",
]
pattern = '|'.join(keywords)
pattern = "|".join(keywords)
regex = re.compile(pattern, re.IGNORECASE)
filtered_list = [item for item in d if not regex.search(item.get('SERVICE', ''))]
filtered_list = [item for item in d if not regex.search(item.get("SERVICE", ""))]
return filtered_list
@@ -229,13 +261,28 @@ def move_percentage_to_rate(df):
Returns:
pandas.DataFrame: The DataFrame with percentage values moved from the 'REIMBURSEMENT_FLAT_FEE' to the 'REIMBURSEMENT_RATE' column.
"""
def move_percentage(value):
if isinstance(value, str) and '%' in value:
if isinstance(value, str) and "%" in value:
return True
return False
df['REIMBURSEMENT_RATE'] = df.apply(lambda row: row['REIMBURSEMENT_FLAT_FEE'] if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_RATE'], axis=1)
df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: None if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1)
df["REIMBURSEMENT_RATE"] = df.apply(
lambda row: (
row["REIMBURSEMENT_FLAT_FEE"]
if move_percentage(row["REIMBURSEMENT_FLAT_FEE"])
else row["REIMBURSEMENT_RATE"]
),
axis=1,
)
df["REIMBURSEMENT_FLAT_FEE"] = df.apply(
lambda row: (
None
if move_percentage(row["REIMBURSEMENT_FLAT_FEE"])
else row["REIMBURSEMENT_FLAT_FEE"]
),
axis=1,
)
return df
@@ -255,12 +302,15 @@ def set_rate_to_zero_if_not_covered(df):
Returns:
pandas.DataFrame: The updated DataFrame with adjusted 'REIMURSEMENT_RATE' values where applicable.
"""
def check_and_set_rate(row):
if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'not covered', row['FULL_METHODOLOGY'], re.IGNORECASE):
if pd.notna(row["FULL_METHODOLOGY"]) and re.search(
r"not covered", row["FULL_METHODOLOGY"], re.IGNORECASE
):
return 0
return row['REIMBURSEMENT_RATE']
df['REIMBURSEMENT_RATE'] = df.apply(check_and_set_rate, axis=1)
return row["REIMBURSEMENT_RATE"]
df["REIMBURSEMENT_RATE"] = df.apply(check_and_set_rate, axis=1)
return df
@@ -279,13 +329,18 @@ def adjust_reimbursement_rate(df):
Returns:
pandas.DataFrame: The updated DataFrame with recalibrated 'REIMBURSEMENT_RATE' values based on the specified methodology condition.
"""
def adjust_rate(row):
if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'case insensitive', row['FULL_METHODOLOGY'], re.IGNORECASE):
if pd.notna(row['REIMBURSEMENT_RATE']) and isinstance(row['REIMBURSEMENT_RATE'], (int, float)):
return 100 - row['REIMBURSEMENT_RATE']
return row['REIMBURSEMENT_RATE']
df['REIMBURSEMENT_RATE'] = df.apply(adjust_rate, axis=1)
if pd.notna(row["FULL_METHODOLOGY"]) and re.search(
r"case insensitive", row["FULL_METHODOLOGY"], re.IGNORECASE
):
if pd.notna(row["REIMBURSEMENT_RATE"]) and isinstance(
row["REIMBURSEMENT_RATE"], (int, float)
):
return 100 - row["REIMBURSEMENT_RATE"]
return row["REIMBURSEMENT_RATE"]
df["REIMBURSEMENT_RATE"] = df.apply(adjust_rate, axis=1)
return df
@@ -310,12 +365,12 @@ def clean_td(td):
for d in td:
new_d = {}
for k, v in d.items():
if 'DATE' in k:
if "DATE" in k:
new_d[k] = v if isinstance(v, list) else [v]
elif k not in ['page_num', 'Filename']:
if isinstance(v, str) and ',' in v:
new_d[k] = [item.strip() for item in v.split(',')]
elif v == 'N/A':
elif k not in ["page_num", "Filename"]:
if isinstance(v, str) and "," in v:
new_d[k] = [item.strip() for item in v.split(",")]
elif v == "N/A":
new_d[k] = []
else:
new_d[k] = [v] if isinstance(v, str) else v
@@ -325,7 +380,6 @@ def clean_td(td):
return td_clean
def sanitize_combined(df):
"""
Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value.
@@ -345,6 +399,7 @@ def sanitize_combined(df):
df[column] = df[column].apply(sanitize_value)
return df
def extract_codes_CPT(value):
"""
Extracts and formats CPT codes from a given input value.
@@ -364,16 +419,17 @@ def extract_codes_CPT(value):
"""
if pd.isna(value):
return value
value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE)
pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b'
value = re.sub(r"\bthrough\b", "-", str(value), flags=re.IGNORECASE)
pattern = r"\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b"
matches = re.findall(pattern, value)
if matches:
return ', '.join([''.join(match) for match in matches])
return ", ".join(["".join(match) for match in matches])
else:
return ""
def extract_codes_Diagnosis(value):
"""
Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats.
@@ -392,14 +448,15 @@ def extract_codes_Diagnosis(value):
"""
if pd.isna(value):
return value
pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b'
pattern = r"\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b"
matches = re.findall(pattern, value)
if matches:
return ', '.join(matches)
return ", ".join(matches)
else:
return ""
def extract_codes_Revenue(value):
"""
Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits.
@@ -417,14 +474,15 @@ def extract_codes_Revenue(value):
"""
if pd.isna(value):
return value
pattern = r'\b\d{3,4}\b'
pattern = r"\b\d{3,4}\b"
matches = re.findall(pattern, value)
if matches:
return ', '.join(matches)
return ", ".join(matches)
else:
return ""
def clean_code_column(df, column_name, extract_function):
"""
Applies a specified function to clean and extract codes from a specific column in a DataFrame.
@@ -443,19 +501,23 @@ def clean_code_column(df, column_name, extract_function):
pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes.
"""
df[column_name] = df[column_name].apply(extract_function)
return df
return df
def clean_dates(df):
for idx, row in df.iterrows():
if row['LOB_PRICING_TERMS_EFFECTIVE_DATE'] == row['LOB_PRICING_TERMS_TERMINATION_DATE']:
df.loc[idx, 'LOB_PRICING_TERMS_TERMINATION_DATE'] = 'N/A'
if (
row["LOB_PRICING_TERMS_EFFECTIVE_DATE"]
== row["LOB_PRICING_TERMS_TERMINATION_DATE"]
):
df.loc[idx, "LOB_PRICING_TERMS_TERMINATION_DATE"] = "N/A"
return df
def add_PG_columns(results_dicts):
for key in list(results_dicts[0].keys()):
if key != 'page_num':
if key != "page_num":
for d in results_dicts:
d[f"{key}_PG"] = d['page_num']
d[f"{key}_PG"] = d["page_num"]
return results_dicts
fieldExtraction/src/preprocess.py
+1 -1
View File
@@ -133,7 +133,7 @@ def clean_billed_charges(contract_text):
"Billed Charges",
"the rates set forth in this Exhibit",
"the rutes set forth in this Exhibit",
"Provider's usual and customary charge"
"Provider's usual and customary charge",
]
max_substring_length = max([len(s) for s in substrings])
fieldExtraction/src/prompts.py
+3
View File
@@ -44,6 +44,7 @@ Write ONLY the value or values in the list that DIRECTLY applies to the Service
If the Service and Methodology are not found on the page, write the LAST item in the list. Do not write any other commentary or explanation.
"""
def BOTTOM_UP_PRIMARY(page, payer):
return f"""
### PAGE START ### {page} ### PAGE END
@@ -221,6 +222,7 @@ Here are the entities, represented as dictionary objects:
Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself.
"""
def TOP_DOWN_METAL_LEVEL(LOB, page):
return f"""### PAGE START ### {page} ### PAGE END
@@ -458,6 +460,7 @@ def get_carveout_list():
"Hematology",
]
def prompt_contract_lob(page):
return f"""### PAGE START ### {page} ### PAGE END
fieldExtraction/src/test.py
+11 -12
View File
@@ -25,21 +25,21 @@ import shutil
# return f"""
# ### PAGE START ### {page} ### PAGE END
# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties.
# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties.
# The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $).
# The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $).
# Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded.
# If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.
# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list.
# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list.
# Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
# 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A.
# 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
# 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
# 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology.
# 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology.
# 'DATE_RANGE' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'.
# 'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A'
@@ -60,19 +60,22 @@ import shutil
# print(answer_dict[page_num])
batch1_files = pd.read_excel("C:\\Users\\kminhas\\Documents\\Doczy\\Facility TINs_DOCZY AI_DOCUMENT EXTRACTION_ASSIGNMENT_7.1.2024.xlsx", sheet_name='DOCZY AI')
batch1_files = batch1_files['File Name']
batch1_files = pd.read_excel(
"C:\\Users\\kminhas\\Documents\\Doczy\\Facility TINs_DOCZY AI_DOCUMENT EXTRACTION_ASSIGNMENT_7.1.2024.xlsx",
sheet_name="DOCZY AI",
)
batch1_files = batch1_files["File Name"]
# print(list(batch1_files))
tdrive_path = "T:\\AArete Client Work\\Centene\\Restricted\\HealthNet\\Doczy Contract Metadata\\1. Contracts\\05232024\\Extracted"
batch_path = os.path.join(tdrive_path, 'Batch0')
batch_path = os.path.join(tdrive_path, "Batch0")
try:
os.mkdir(batch_path)
except:
pass
for file in os.listdir(tdrive_path):
if file.endswith('.Pdf') and file.split('.Pdf')[0] in list(batch1_files):
if file.endswith(".Pdf") and file.split(".Pdf")[0] in list(batch1_files):
try:
source_path = os.path.join(tdrive_path, file)
destination_path = os.path.join(batch_path, file)
@@ -80,7 +83,3 @@ for file in os.listdir(tdrive_path):
print(f"Moved file: {file}")
except:
pass
fieldExtraction/src/utils.py
+3 -1
View File
@@ -139,7 +139,9 @@ def consolidate_csvs(
pass
concatenated_df = pd.concat(df_list, ignore_index=True)
concatenated_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False)
concatenated_df.to_csv(
os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False
)
print(f"All CSV files have been consolidated into {output_file}")