Katon Minhas
64e754d40f
Refactor/hotfix refactor * Initial hotfix refactor * Fix references * Remove test Approved-by: Alex Galarce
347 lines
15 KiB
Python
347 lines
15 KiB
Python
import ast
|
|
import re
|
|
|
|
import pandas as pd
|
|
|
|
import postprocessing_funcs
|
|
from regex_funcs import find_regex_matches
|
|
from valid import STATE_MAP
|
|
|
|
|
|
def convert_to_list(value):
|
|
if isinstance(value, str):
|
|
# Remove any trailing commas or spaces
|
|
value = value.strip().replace(',', ' ') # Replace commas with spaces
|
|
|
|
# Check if the string is in the form of a list and safely convert it
|
|
if value.startswith('[') and value.endswith(']'):
|
|
try:
|
|
return ast.literal_eval(value) # Safely convert string to list
|
|
except:
|
|
return value.split() # Fallback to split by spaces if literal_eval fails
|
|
else:
|
|
# Split by spaces and handle both alphanumeric and numeric values
|
|
return value.split() # Just split by spaces, treat everything as a string
|
|
elif isinstance(value, list):
|
|
return value # If it's already a list, return as is
|
|
else:
|
|
return [value] # Wrap non-string, non-list types in a list
|
|
|
|
|
|
def list_to_string(lst):
|
|
if not lst: # Check if the list is empty
|
|
return None
|
|
else:
|
|
return ", ".join(lst) # Join elements with commas
|
|
|
|
def remove_alphabets(input_str):
|
|
# Check if the string contains any alphabetic characters
|
|
if any(char.isalpha() for char in input_str):
|
|
return None # Return None if there are alphabets
|
|
else:
|
|
return input_str # Return the string if no alphabets are present
|
|
|
|
|
|
def clean_output_postprocess(merged_df):
|
|
merged_df['dummy'] = merged_df['IRS #']
|
|
merged_df['dummy'] = merged_df['dummy'].astype(str)
|
|
merged_df['dummy'] = merged_df['dummy'].apply(convert_to_list)
|
|
merged_df['dummy'] = merged_df['dummy'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
|
merged_df['dummy'] = merged_df['dummy'].apply(lambda lst: [x for x in [postprocessing_funcs.add_hyphen_if_needed(x) for x in lst] if x is not None])
|
|
merged_df['dummy'] = merged_df['dummy'].apply(list_to_string)
|
|
|
|
if 'PROV_TIN_OTHER' in merged_df.columns:
|
|
merged_df['dummy_other'] = merged_df['PROV_TIN_OTHER']
|
|
merged_df['dummy_other'] = merged_df['dummy_other'].astype(str)
|
|
merged_df['dummy_other'] = merged_df['dummy_other'].apply(convert_to_list)
|
|
merged_df['dummy_other'] = merged_df['dummy_other'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
|
merged_df['dummy_other'] = merged_df['dummy_other'].apply(lambda lst: [x for x in [postprocessing_funcs.add_hyphen_if_needed(x) for x in lst] if x is not None])
|
|
merged_df['dummy_other'] = merged_df['dummy_other'].apply(list_to_string)
|
|
|
|
if 'PROV_GROUP_TIN_SIGNATORY' in merged_df.columns:
|
|
merged_df['dummy_other_signatory'] = merged_df["PROV_GROUP_TIN_SIGNATORY"]
|
|
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].astype(str)
|
|
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(convert_to_list)
|
|
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
|
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(lambda lst: [x for x in [postprocessing_funcs.add_hyphen_if_needed(x) for x in lst] if x is not None])
|
|
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(list_to_string)
|
|
|
|
return merged_df
|
|
|
|
|
|
def check_tin_in_filename(filename):
|
|
"""
|
|
Checks for the pattern match in filename
|
|
|
|
Args:
|
|
filename(str): Name of the file
|
|
Returns:
|
|
tin_from_title(str): regex match for patterns in filename
|
|
"""
|
|
patterns = [r"\b\d{2}-\d{7}\b", r"\b\d{9}\b"]
|
|
for pattern in patterns:
|
|
tin_from_title = re.findall(pattern, filename)
|
|
if len(tin_from_title) == 1:
|
|
tin_from_title = [postprocessing_funcs.add_hyphen_if_needed(s) for s in tin_from_title]
|
|
return tin_from_title[0]
|
|
return 'empty'
|
|
|
|
|
|
def signature_page_irs(text_dict, page_list, matches, pattern, filename):
|
|
"""
|
|
In case of multiple matches, checks if a TIN is on the Signature page
|
|
|
|
Args:
|
|
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
|
page_list(list): List of pages where the match was found
|
|
matches(list): List of regex match(s)
|
|
Returns
|
|
provider_tin(str): Provider group TIN - either from signature page or filename
|
|
other_tins(str): Remaining TINs
|
|
"""
|
|
# loops through regex matches and thier pages to check for signature page
|
|
tin_on_signature_page = []
|
|
for tin in matches:
|
|
for page in page_list:
|
|
if tin in text_dict[page] and 'signature' in text_dict[page]:
|
|
tin_on_signature_page.append(tin)
|
|
tin_on_signature_page = list(set(tin_on_signature_page))
|
|
|
|
#adds hyphen if needed
|
|
matches = [postprocessing_funcs.add_hyphen_if_needed(s) for s in matches]
|
|
tin_on_signature_page = [postprocessing_funcs.add_hyphen_if_needed(s) for s in tin_on_signature_page]
|
|
|
|
#case1 - If 1 TIN found on signature page, return it, put the remaining ones in "TIN Others" have signature_tin = N/A
|
|
if len(tin_on_signature_page) == 1:
|
|
provider_tin = tin_on_signature_page[0]
|
|
signature_tin = "N/A"
|
|
matches = [item for item in matches if item not in tin_on_signature_page]
|
|
other_tins = ", ".join(matches)
|
|
return provider_tin, signature_tin, other_tins
|
|
|
|
#case2 - No TIN found on the signature page, check in filename
|
|
elif len(tin_on_signature_page) == 0:
|
|
provider_tin = check_tin_in_filename(filename)
|
|
if provider_tin == 'empty':
|
|
provider_tin = "N/A"
|
|
signature_tin = "N/A"
|
|
other_tins = [item for item in matches if item != provider_tin]
|
|
other_tins = ", ".join(other_tins)
|
|
return provider_tin, signature_tin, other_tins
|
|
|
|
#case3 - More than 1 TIN on signature page - check for TIN in file name, let the others go in signatory_tins
|
|
# and finally check for TINs not on signature page but in "matches" list to add them to "tin_others"
|
|
else:
|
|
filename_tin_check = check_tin_in_filename(filename)
|
|
if filename_tin_check == 'empty':
|
|
provider_tin = "N/A"
|
|
signature_tin = ", ".join(tin_on_signature_page)
|
|
other_tins = "N/A"
|
|
return provider_tin, signature_tin, other_tins
|
|
else:
|
|
provider_tin = filename_tin_check
|
|
signature_tin = [item for item in tin_on_signature_page if item != provider_tin]
|
|
signature_tin = ", ".join(signature_tin)
|
|
|
|
if len(matches)> len(tin_on_signature_page):
|
|
other_tins = [item for item in matches if item not in tin_on_signature_page and item != provider_tin]
|
|
other_tins = ", ".join(other_tins)
|
|
return provider_tin, signature_tin, other_tins
|
|
else:
|
|
other_tins = "N/A"
|
|
return provider_tin, signature_tin, other_tins
|
|
|
|
|
|
def irs_hotfix(filename, text_dict: dict[str, str]):
|
|
"""Returns dictionery with Provider and other TIN(s) based on regex matches and presence on Signatory page
|
|
|
|
Args:
|
|
filename: Name of the contract
|
|
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
|
|
|
Returns:
|
|
TIN_dict(dict): A dictionery with keys 'PROV_GROUP_TIN' & 'PROV_TIN_OTHER' and values as respective TIN(s)
|
|
"""
|
|
TIN_dict = {}
|
|
|
|
patterns = [r"\b\d{2}-\d{7}\b",
|
|
r"\b\d{9}\b"]
|
|
for pattern in patterns:
|
|
matches, page_list = find_regex_matches(pattern, text_dict)
|
|
matches = list(set(matches))
|
|
if matches:
|
|
if len(matches) == 1:
|
|
matches = [postprocessing_funcs.add_hyphen_if_needed(s) for s in matches]
|
|
TIN_dict["PROV_GROUP_TIN"] = matches[0]
|
|
TIN_dict["PROV_TIN_OTHER"] = "N/A"
|
|
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = "N/A"
|
|
return TIN_dict
|
|
|
|
elif len(matches) >= 1:
|
|
irs_tin, signatory_tin, other_tins = signature_page_irs(text_dict, page_list, matches, pattern, filename)
|
|
TIN_dict["PROV_GROUP_TIN"] = irs_tin
|
|
TIN_dict["PROV_TIN_OTHER"] = other_tins
|
|
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = signatory_tin
|
|
return TIN_dict
|
|
|
|
# If no matches were found in the text_dict, check the filename
|
|
for pattern in patterns:
|
|
tin_from_title = check_tin_in_filename(filename)
|
|
TIN_dict["PROV_TIN_OTHER"] = "N/A"
|
|
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = "N/A"
|
|
if tin_from_title == "empty":
|
|
TIN_dict["PROV_GROUP_TIN"] = "N/A"
|
|
else:
|
|
TIN_dict["PROV_GROUP_TIN"] = tin_from_title
|
|
return TIN_dict
|
|
|
|
def check_field_for_matches(text, valid_lists):
|
|
"""
|
|
Check a field for matches from the valid lists using case-insensitive regex
|
|
Returns tuple of (matched_term, list_type) or (None, None)
|
|
"""
|
|
if pd.isna(text) or text == "":
|
|
return None, None
|
|
text = str(text).lower()
|
|
for list_type, valid_list in valid_lists.items():
|
|
for term in valid_list:
|
|
if re.search(rf'\b{re.escape(term.lower())}\b', text):
|
|
return term, list_type
|
|
return None, None
|
|
|
|
##### NPI #####
|
|
def find_10_digit_numbers(row):
|
|
matches = re.findall(r'\b\d{10}\b', row)
|
|
return matches
|
|
|
|
|
|
def signature_page_npi(text_dict, page_list, matches):
|
|
npi_on_signature_page = []
|
|
for npi in matches:
|
|
for page in page_list:
|
|
if npi in text_dict[page] and 'signature' in text_dict[page]:
|
|
npi_on_signature_page.append(npi)
|
|
npi_on_signature_page = list(set(npi_on_signature_page))
|
|
|
|
if len(npi_on_signature_page) == 1:
|
|
npi = npi_on_signature_page[0]
|
|
npi_other = [item for item in matches if item not in npi_on_signature_page]
|
|
npi_other = ", ".join(npi_other)
|
|
return npi, npi_other
|
|
elif len(npi_on_signature_page) == 0:
|
|
npi = "N/A"
|
|
npi_other = ", ".join(matches)
|
|
return npi, npi_other
|
|
|
|
else:
|
|
npi = ", ".join(npi_on_signature_page)
|
|
npi_other = [item for item in matches if item not in npi_on_signature_page]
|
|
if len(npi_other) == 0:
|
|
npi_other = "N/A"
|
|
return npi, npi_other
|
|
else:
|
|
npi_other = ", ".join(npi_other)
|
|
return npi, npi_other
|
|
|
|
|
|
def npi_hotfix(text_dict: dict[str, str]):
|
|
"""Returns dictionery with Provider and other TIN(s) based on regex matches and presence on Signatory page
|
|
|
|
Args:
|
|
filename: Name of the contract
|
|
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
|
|
|
Returns:
|
|
TIN_dict(dict): A dictionery with keys 'PROV_GROUP_TIN' & 'PROV_TIN_OTHER' and values as respective TIN(s)
|
|
"""
|
|
NPI_dict = {}
|
|
|
|
patterns = [r"\b\d{10}\b"]
|
|
|
|
for pattern in patterns:
|
|
matches, page_list = find_regex_matches(pattern, text_dict)
|
|
matches = [s for s in matches if s[0] in ('1', '2')]
|
|
matches = list(set(matches))
|
|
if len(matches)==1:
|
|
NPI_dict["NPI"] = matches[0]
|
|
NPI_dict["NPI_other"] = "N/A"
|
|
elif len(matches) == 0:
|
|
NPI_dict["NPI"] = "N/A"
|
|
NPI_dict["NPI_other"] = "N/A"
|
|
else:
|
|
npi, npi_other = signature_page_npi(text_dict, page_list, matches)
|
|
NPI_dict["NPI"] = npi
|
|
NPI_dict["NPI_other"] = npi_other
|
|
return NPI_dict
|
|
|
|
|
|
##################################################################################################################################
|
|
|
|
# remove '-' from numbers and drop numbers with more than 2 '-' in them
|
|
def remove_dashes(lst):
|
|
processed_list = []
|
|
for item in lst:
|
|
# If the item contains more than 2 dashes, remove it
|
|
if item.count('-') > 2:
|
|
continue
|
|
else:
|
|
# Remove dashes and keep the digits only
|
|
processed_list.append(item.replace('-', ''))
|
|
|
|
return processed_list
|
|
|
|
##################################################################################################################################
|
|
|
|
def filter_elements_by_length(lst):
|
|
return [item for item in lst if 7 <= len(item) <= 10]
|
|
|
|
##################################################################################################################################
|
|
# Function to filter out non-digit elements from a list
|
|
def filter_non_digits(lst):
|
|
return [item for item in lst if item.isdigit()] # Only keep items that are digits
|
|
|
|
##################################################################################################################################
|
|
|
|
def filter_start_with_1_or_2(lst):
|
|
return [item for item in lst if item.startswith('1') or item.startswith('2')]
|
|
|
|
|
|
##################################################################################################################################
|
|
def convert_to_string(value):
|
|
if pd.isna(value):
|
|
# If the value is NaN, return it as is
|
|
return str(value)
|
|
elif isinstance(value, float):
|
|
# If it's a float, convert to int and then to string to avoid decimal point
|
|
return str(int(value)) if value == value else 'nan'
|
|
return str(value)
|
|
##################################################################################################################################
|
|
|
|
def npi_post_process(df_merged):
|
|
df_merged['dummy'] = df_merged['NPI (10-digits)']
|
|
df_merged['dummy'] = df_merged['dummy'].apply(convert_to_string)
|
|
df_merged['dummy'] = df_merged['dummy'].apply(convert_to_list)
|
|
df_merged['dummy'] = df_merged['dummy'].apply(remove_dashes)
|
|
df_merged['dummy'] = df_merged['dummy'].apply(filter_elements_by_length)
|
|
df_merged['dummy'] = df_merged['dummy'].apply(filter_non_digits)
|
|
df_merged['dummy'] = df_merged['dummy'].apply(filter_start_with_1_or_2)
|
|
df_merged['dummy'] = df_merged['dummy'].apply(list_to_string)
|
|
|
|
if 'PROV_NPI_OTHER' in df_merged.columns:
|
|
df_merged['dummy_other'] = df_merged["PROV_NPI_OTHER"]
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].astype(str)
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].apply(convert_to_list)
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].apply(remove_dashes)
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_elements_by_length)
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_non_digits)
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_start_with_1_or_2)
|
|
df_merged['dummy_other'] = df_merged['dummy_other'].apply(list_to_string)
|
|
|
|
return df_merged
|
|
|
|
|
|
|
|
|
|
|
|
|