462 lines
21 KiB
Python
462 lines
21 KiB
Python
|
|
||
|
|
import pandas as pd
|
||
|
|
import re
|
||
|
|
import difflib
|
||
|
|
|
||
|
|
import config
|
||
|
|
|
||
|
|
def sanitize_value(value):
|
||
|
|
"""
|
||
|
|
Sanitizes a given value by handling nulls, lists, and strings to ensure consistency in formatting.
|
||
|
|
|
||
|
|
This function performs several cleanliness checks and transformations:
|
||
|
|
- It leaves null (NaN) values unchanged.
|
||
|
|
- Converts lists into comma-separated strings.
|
||
|
|
- Strips unnecessary characters from strings and handles strings that represent lists, ensuring individual elements are properly formatted.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
value (varied types): The value to be sanitized, which can be of any type including nulls, lists, or strings.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
varied types: The sanitized value, which can be a string or the original type if no changes were needed.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
if isinstance(value, list):
|
||
|
|
return ', '.join(str(v) for v in value)
|
||
|
|
elif isinstance(value, str):
|
||
|
|
value = value.strip('[]')
|
||
|
|
return ', '.join([item.strip(" '") for item in value.split(',')])
|
||
|
|
elif pd.isna(value):
|
||
|
|
return "N/A"
|
||
|
|
except:
|
||
|
|
return value
|
||
|
|
|
||
|
|
def exact_match(val, valid_values):
|
||
|
|
"""
|
||
|
|
Compares a given string value against a list of valid values to determine if there is an exact match, case-insensitively.
|
||
|
|
|
||
|
|
This function processes the input value by trimming whitespace and converting to uppercase for a case-insensitive comparison.
|
||
|
|
It then checks each item in the list of valid values (also treated case-insensitively) to find an exact match. If a match is found,
|
||
|
|
the function returns the valid value in its original form as specified in the list. If no match is found, it returns None.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
val (str): The value to be checked for an exact match.
|
||
|
|
valid_values (list of str): A list of valid values against which the input value is compared.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
str or None: The matching valid value if found, or None if no exact match is found.
|
||
|
|
"""
|
||
|
|
val = val.strip().upper()
|
||
|
|
for valid_val in valid_values:
|
||
|
|
if val == valid_val.upper():
|
||
|
|
return valid_val
|
||
|
|
return None
|
||
|
|
|
||
|
|
def clean_columns_combined(df, column_name, valid_values, new_column_name):
|
||
|
|
"""
|
||
|
|
Cleans and standardizes entries in a specified column of a DataFrame based on a list of valid values, creating a new column with standardized values.
|
||
|
|
|
||
|
|
This function first sanitizes the values in the specified column. It then examines each entry in that column,
|
||
|
|
looking for any sub-values (separated by commas) that exactly match any of the provided valid values. If a match is found,
|
||
|
|
it is recorded in a new column. The function also tracks changes and compares the original and cleaned unique values.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the data to be cleaned.
|
||
|
|
column_name (str): The name of the column to be cleaned.
|
||
|
|
valid_values (list of str): A list of valid values to be used for checking the entries in the specified column.
|
||
|
|
new_column_name (str): The name of the new column where cleaned, standardized values will be stored.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
tuple: Contains arrays of original unique values, cleaned unique values, and a dictionary of changes from original to new values.
|
||
|
|
"""
|
||
|
|
changes = {}
|
||
|
|
df[column_name] = df[column_name].apply(sanitize_value)
|
||
|
|
original_values = df[column_name].unique()
|
||
|
|
|
||
|
|
def update_column(entry):
|
||
|
|
if pd.notna(entry):
|
||
|
|
terms = entry.split(',')
|
||
|
|
for term in terms:
|
||
|
|
match = exact_match(term, valid_values)
|
||
|
|
if match:
|
||
|
|
return match
|
||
|
|
return None
|
||
|
|
|
||
|
|
df[new_column_name] = df[column_name].apply(update_column)
|
||
|
|
cleaned_values = df[new_column_name].unique()
|
||
|
|
return original_values, cleaned_values, changes
|
||
|
|
|
||
|
|
|
||
|
|
def get_closest_match(val, valid_values, similarity_threshold=0.7):
|
||
|
|
"""
|
||
|
|
Finds the closest match for a given string from a list of valid values, using a similarity threshold.
|
||
|
|
|
||
|
|
This function cleans the input value and compares it to each cleaned value in the valid values list using a fuzzy
|
||
|
|
matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no
|
||
|
|
matches meet the threshold, the function returns None.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
val (str): The value to be matched.
|
||
|
|
valid_yvalues (list of str): A list of valid values against which the input value is compared.
|
||
|
|
similarity_threshold (float): The cutoff similarity score (between 0 and 1) below which matches are not considered.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
str or None: The closest valid value if a match is found; otherwise, None.
|
||
|
|
"""
|
||
|
|
if pd.isna(val):
|
||
|
|
return None
|
||
|
|
val = val.strip().upper()
|
||
|
|
matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold)
|
||
|
|
return matches[0] if matches else None
|
||
|
|
|
||
|
|
|
||
|
|
def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
|
||
|
|
"""
|
||
|
|
Applies fuzzy matching to entries in a specified column of a DataFrame to standardize them according to a list of valid values,
|
||
|
|
using a defined similarity threshold to determine matches.
|
||
|
|
|
||
|
|
The function sanitizes the column values and then processes each entry, splitting it into sub-values if multiple are present.
|
||
|
|
Each sub-value is compared against the valid values list using a fuzzy matching algorithm. If a close match is found (above the
|
||
|
|
threshold), it replaces the original sub-value. The function logs any changes made for tracking and verification purposes.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the column to be cleaned.
|
||
|
|
column_name (str): The name of the column to be cleaned.
|
||
|
|
valid_values (list of str): A list of valid values used for fuzzy matching.
|
||
|
|
threshold (float): The similarity threshold for accepting matches.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
tuple: Contains arrays of original unique values, cleaned unique values, and a dictionary of changes (original to new values).
|
||
|
|
"""
|
||
|
|
changes = {}
|
||
|
|
df[column_name] = df[column_name].apply(sanitize_value)
|
||
|
|
original_values = df[column_name].unique()
|
||
|
|
|
||
|
|
def log_and_clean(entry):
|
||
|
|
if pd.notna(entry):
|
||
|
|
words = entry.split(',')
|
||
|
|
cleaned_words = []
|
||
|
|
for word in words:
|
||
|
|
cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
|
||
|
|
if cleaned_word and word.strip().upper() != cleaned_word:
|
||
|
|
changes[word.strip()] = cleaned_word
|
||
|
|
cleaned_words.append(cleaned_word if cleaned_word else word.strip())
|
||
|
|
return ', '.join(cleaned_words)
|
||
|
|
return None
|
||
|
|
|
||
|
|
df[column_name] = df[column_name].apply(log_and_clean)
|
||
|
|
cleaned_values = df[column_name].unique()
|
||
|
|
return original_values, cleaned_values, changes
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def correct_misplaced_values(df, columns, valid_values_dict):
|
||
|
|
"""
|
||
|
|
Corrects misplaced values within specified columns of a DataFrame based on a dictionary of valid values for each column.
|
||
|
|
|
||
|
|
This function iterates through each row of the DataFrame and checks the values of specified columns. If a value
|
||
|
|
is found that matches valid entries for a different column (based on the valid_values_dict), it moves the value to
|
||
|
|
the appropriate column. If the target column already contains a value, it uses fuzzy matching to determine if the
|
||
|
|
misplaced value should replace the existing one. If a replacement isn't suitable, it logs the correction attempt.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the data to be corrected.
|
||
|
|
columns (list of str): The columns to check for misplaced values.
|
||
|
|
valid_values_dict (dict): A dictionary mapping column names to lists of valid values for those columns.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
pandas.DataFrame: The DataFrame with corrected placements of values across the specified columns.
|
||
|
|
"""
|
||
|
|
for index, row in df.iterrows():
|
||
|
|
for col in columns:
|
||
|
|
if pd.notna(row[col]):
|
||
|
|
terms = row[col].split(',')
|
||
|
|
for term in terms:
|
||
|
|
term = term.strip()
|
||
|
|
for target_col, valid_values in valid_values_dict.items():
|
||
|
|
if target_col != col:
|
||
|
|
match = exact_match(term, valid_values)
|
||
|
|
if match:
|
||
|
|
if pd.isna(row[target_col]) or not row[target_col].strip():
|
||
|
|
df.at[index, target_col] = match
|
||
|
|
df.at[index, col] = None
|
||
|
|
else:
|
||
|
|
current_value = row[target_col].strip()
|
||
|
|
if get_closest_match(match, [current_value], config.FUZZY_MATCH_THRESHOLD) is None:
|
||
|
|
df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell"
|
||
|
|
df.at[index, col] = None
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
def filter_service_column(d):
|
||
|
|
"""
|
||
|
|
Filters out items in a list of dictionaries based on specified keywords within the 'SERVICE' key.
|
||
|
|
|
||
|
|
This function uses a list of predefined keywords associated with non-service related terms like 'LIABILITY' or 'RISK'.
|
||
|
|
It constructs a regular expression pattern from these keywords and removes any dictionary items where the 'SERVICE' key contains
|
||
|
|
any of these keywords, effectively filtering the list to exclude items that are likely to be irrelevant or misclassified
|
||
|
|
as services when they pertain to other contractual obligations or conditions.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
d (list): A list of dictionaries, each containing a 'SERVICE' key among others.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
list: A new list of dictionaries with items containing specified keywords in the 'SERVICE' key removed.
|
||
|
|
"""
|
||
|
|
keywords = [
|
||
|
|
'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED',
|
||
|
|
'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES',
|
||
|
|
]
|
||
|
|
pattern = '|'.join(keywords)
|
||
|
|
regex = re.compile(pattern, re.IGNORECASE)
|
||
|
|
|
||
|
|
filtered_list = [item for item in d if not regex.search(item.get('SERVICE', ''))]
|
||
|
|
|
||
|
|
return filtered_list
|
||
|
|
|
||
|
|
|
||
|
|
def move_percentage_to_rate(df):
|
||
|
|
"""
|
||
|
|
Moves percentage values from the 'REIMBURSEMENT_FLAT_FEE' column to the 'REIMBURSEMENT_RATE' column in a DataFrame.
|
||
|
|
|
||
|
|
This function identifies entries in the 'REIMBURSEMENT_FLAT_FEE' column that contain a percentage sign, indicating they are
|
||
|
|
incorrectly placed. It then transfers these percentage values to the 'REIMBURSEMENT_RATE' column. The original 'REIMBURSEMENT_FLAT_FEE'
|
||
|
|
entries from which percentages are moved are set to None, correcting the placement of data within the DataFrame.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the reimbursement data to be adjusted.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
pandas.DataFrame: The DataFrame with percentage values moved from the 'REIMBURSEMENT_FLAT_FEE' to the 'REIMBURSEMENT_RATE' column.
|
||
|
|
"""
|
||
|
|
def move_percentage(value):
|
||
|
|
if isinstance(value, str) and '%' in value:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
df['REIMBURSEMENT_RATE'] = df.apply(lambda row: row['REIMBURSEMENT_FLAT_FEE'] if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_RATE'], axis=1)
|
||
|
|
df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: None if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1)
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
def set_rate_to_zero_if_not_covered(df):
|
||
|
|
"""
|
||
|
|
Sets the 'REIMBURSEMENT_RATE' to zero for entries in the DataFrame where the service is explicitly not covered,
|
||
|
|
as indicated by the content of the 'FULL_METHODOLOGY' column.
|
||
|
|
|
||
|
|
This function checks each row's 'FULL_METHODOLOGY' column for phrases indicating that the service is not covered
|
||
|
|
(e.g., "not covered"). If such a phrase is found, the 'REIMBURSEMENT_RATE' for that row is set to zero to reflect that
|
||
|
|
there is no financial reimbursement for these services. This helps ensure that the data accurately represents the terms
|
||
|
|
of service coverage.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the reimbursement and methodology data.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
pandas.DataFrame: The updated DataFrame with adjusted 'REIMURSEMENT_RATE' values where applicable.
|
||
|
|
"""
|
||
|
|
def check_and_set_rate(row):
|
||
|
|
if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'not covered', row['FULL_METHODOLOGY'], re.IGNORECASE):
|
||
|
|
return 0
|
||
|
|
return row['REIMBURSEMENT_RATE']
|
||
|
|
|
||
|
|
df['REIMBURSEMENT_RATE'] = df.apply(check_and_set_rate, axis=1)
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
def adjust_reimbursement_rate(df):
|
||
|
|
"""
|
||
|
|
Adjusts the 'REIMBURSEMENT_RATE' in the DataFrame based on specific conditions mentioned in the 'FULL_METHODOLOGY' column.
|
||
|
|
|
||
|
|
This function reviews each row's 'FULL_METHODOLOGY' column for mentions of being "case insensitive" as a part of the methodology.
|
||
|
|
If this condition is noted and the 'REIMBURSEMENT_RATE' is a numerical value, it adjusts the rate by subtracting it from 100,
|
||
|
|
effectively transforming the rate into a complementary percentage. This modification is intended to correct or recalibrate
|
||
|
|
reimbursement rates under specific contractual terms.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the methodology and reimbursement rate data.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
pandas.DataFrame: The updated DataFrame with recalibrated 'REIMBURSEMENT_RATE' values based on the specified methodology condition.
|
||
|
|
"""
|
||
|
|
def adjust_rate(row):
|
||
|
|
if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'case insensitive', row['FULL_METHODOLOGY'], re.IGNORECASE):
|
||
|
|
if pd.notna(row['REIMBURSEMENT_RATE']) and isinstance(row['REIMBURSEMENT_RATE'], (int, float)):
|
||
|
|
return 100 - row['REIMBURSEMENT_RATE']
|
||
|
|
return row['REIMBURSEMENT_RATE']
|
||
|
|
|
||
|
|
df['REIMBURSEMENT_RATE'] = df.apply(adjust_rate, axis=1)
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
def clean_td(td):
|
||
|
|
"""
|
||
|
|
Cleans the data structure returned from a 'Top Down' analysis by normalizing and formatting its values.
|
||
|
|
|
||
|
|
This function iterates through each entry in the list of dictionaries returned from the top down analysis,
|
||
|
|
modifying data based on specific criteria:
|
||
|
|
- Dates are converted to lists if not already in list form.
|
||
|
|
- Comma-separated strings are split into lists.
|
||
|
|
- 'N/A' values are replaced with empty lists.
|
||
|
|
- Preserves specific keys as they are (e.g., 'page_num', 'Filename').
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
td (list of dict): The list of dictionaries containing the results from top down analysis.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
list of dict: A new list of dictionaries with the cleaned and normalized data.
|
||
|
|
"""
|
||
|
|
td_clean = []
|
||
|
|
for d in td:
|
||
|
|
new_d = {}
|
||
|
|
for k, v in d.items():
|
||
|
|
if 'DATE' in k:
|
||
|
|
new_d[k] = v if isinstance(v, list) else [v]
|
||
|
|
elif k not in ['page_num', 'Filename']:
|
||
|
|
if isinstance(v, str) and ',' in v:
|
||
|
|
new_d[k] = [item.strip() for item in v.split(',')]
|
||
|
|
elif v == 'N/A':
|
||
|
|
new_d[k] = []
|
||
|
|
else:
|
||
|
|
new_d[k] = [v] if isinstance(v, str) else v
|
||
|
|
else:
|
||
|
|
new_d[k] = v
|
||
|
|
td_clean.append(new_d)
|
||
|
|
return td_clean
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def sanitize_combined(df):
|
||
|
|
"""
|
||
|
|
Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value.
|
||
|
|
|
||
|
|
This function iterates over each column in the DataFrame, applying a sanitization function to each element.
|
||
|
|
The sanitization function is designed to clean or adjust the data in a way that makes it more consistent or appropriate
|
||
|
|
for further processing or analysis. This could include operations like trimming whitespace, correcting formats,
|
||
|
|
or removing unwanted characters.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame to be sanitized.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
pandas.DataFrame: The sanitized DataFrame, with the same structure but cleaned data.
|
||
|
|
"""
|
||
|
|
for column in df.columns:
|
||
|
|
df[column] = df[column].apply(sanitize_value)
|
||
|
|
return df
|
||
|
|
|
||
|
|
def extract_codes_CPT(value):
|
||
|
|
"""
|
||
|
|
Extracts and formats CPT codes from a given input value.
|
||
|
|
|
||
|
|
This function searches for patterns that match CPT codes within the provided input. CPT codes typically consist
|
||
|
|
of a letter followed by four digits, or just five digits, and may include a two-letter state code as a suffix.
|
||
|
|
The function handles cases where the range of codes is specified using 'through' by converting it to a hyphen.
|
||
|
|
If any codes are found, they are returned as a comma-separated string. If no codes are found or the input is null,
|
||
|
|
an appropriate value (empty string or null) is returned.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
value (str or NaN): The value from which to extract CPT codes, which could potentially be a string or NaN.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
str: A comma-separated string of the extracted CPT codes, or an empty string if none are found.
|
||
|
|
If the input is NaN, the function returns the input as is.
|
||
|
|
"""
|
||
|
|
if pd.isna(value):
|
||
|
|
return value
|
||
|
|
|
||
|
|
value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE)
|
||
|
|
|
||
|
|
pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b'
|
||
|
|
matches = re.findall(pattern, value)
|
||
|
|
if matches:
|
||
|
|
return ', '.join([''.join(match) for match in matches])
|
||
|
|
else:
|
||
|
|
return ""
|
||
|
|
|
||
|
|
def extract_codes_Diagnosis(value):
|
||
|
|
"""
|
||
|
|
Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats.
|
||
|
|
|
||
|
|
This function identifies patterns consistent with ICD diagnosis codes, which are generally a letter followed by numbers
|
||
|
|
and may include additional alphanumeric characters and a period for further specification. The function handles null inputs
|
||
|
|
gracefully, returning the input itself if null. If valid diagnosis codes are found, they are returned as a comma-separated
|
||
|
|
string. If no codes are found, the function returns an empty string.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
value (str or NaN): The input value from which to extract diagnosis codes, which could be a string or NaN.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
str: A comma-separated string of the extracted diagnosis codes, or an empty string if none are found.
|
||
|
|
If the input is NaN, the function returns the input as is.
|
||
|
|
"""
|
||
|
|
if pd.isna(value):
|
||
|
|
return value
|
||
|
|
|
||
|
|
pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b'
|
||
|
|
matches = re.findall(pattern, value)
|
||
|
|
if matches:
|
||
|
|
return ', '.join(matches)
|
||
|
|
else:
|
||
|
|
return ""
|
||
|
|
|
||
|
|
def extract_codes_Revenue(value):
|
||
|
|
"""
|
||
|
|
Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits.
|
||
|
|
|
||
|
|
This function searches for patterns that match revenue codes in the input provided. It handles cases where the input might be null (NaN),
|
||
|
|
returning the input itself in such cases. If any valid revenue codes are found, they are formatted into a comma-separated string.
|
||
|
|
If no codes are found, the function returns an empty string, indicating the absence of recognizable codes in the input.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
value (str or NaN): The input value from which to extract revenue codes, which could be a string or NaN.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
str: A comma-separated string of the extracted revenue codes, or an empty string if none are found.
|
||
|
|
If the input is NaN, the function returns the input as is.
|
||
|
|
"""
|
||
|
|
if pd.isna(value):
|
||
|
|
return value
|
||
|
|
|
||
|
|
pattern = r'\b\d{3,4}\b'
|
||
|
|
matches = re.findall(pattern, value)
|
||
|
|
if matches:
|
||
|
|
return ', '.join(matches)
|
||
|
|
else:
|
||
|
|
return ""
|
||
|
|
|
||
|
|
def clean_code_column(df, column_name, extract_function):
|
||
|
|
"""
|
||
|
|
Applies a specified function to clean and extract codes from a specific column in a DataFrame.
|
||
|
|
|
||
|
|
This function takes a DataFrame and applies a given extraction function to each element of a specified column.
|
||
|
|
The extraction function is expected to format or extract codes (e.g., CPT, ICD, Revenue codes) from the column's values.
|
||
|
|
This cleaned and extracted data replaces the original values in the column. The function is flexible and can be used
|
||
|
|
with any function designed to process and return cleaned values from a single input.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
df (pandas.DataFrame): The DataFrame containing the data to be cleaned.
|
||
|
|
column_name (str): The name of the column to be cleaned.
|
||
|
|
extract_function (function): The function to apply to each value in the specified column, which processes and returns the cleaned value.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes.
|
||
|
|
"""
|
||
|
|
df[column_name] = df[column_name].apply(extract_function)
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
def clean_dates(df):
|
||
|
|
for idx, row in df.iterrows():
|
||
|
|
if row['LOB_PRICING_TERMS_EFFECTIVE_DATE'] == row['LOB_PRICING_TERMS_TERMINATION_DATE']:
|
||
|
|
df.loc[idx, 'LOB_PRICING_TERMS_TERMINATION_DATE'] = 'N/A'
|
||
|
|
|
||
|
|
return df
|
||
|
|
|
||
|
|
def add_PG_columns(results_dicts):
|
||
|
|
for key in list(results_dicts[0].keys()):
|
||
|
|
if key != 'page_num':
|
||
|
|
for d in results_dicts:
|
||
|
|
d[f"{key}_PG"] = d['page_num']
|
||
|
|
return results_dicts
|