96 lines
3.6 KiB
Python
96 lines
3.6 KiB
Python
|
import pandas as pd
|
||
|
|
import re
|
||
|
|
import difflib
|
||
|
|
|
||
|
|
def sanitize_value(value):
|
||
|
|
""" Remove brackets from list items and clean the values. """
|
||
|
|
if pd.isna(value):
|
||
|
|
return value
|
||
|
|
if isinstance(value, str):
|
||
|
|
value = value.strip('[]')
|
||
|
|
if ',' in value:
|
||
|
|
value = ', '.join([item.strip(" '") for item in value.split(',')])
|
||
|
|
else:
|
||
|
|
value = value.strip(" '")
|
||
|
|
return value
|
||
|
|
|
||
|
|
def exact_match(val, valid_values):
|
||
|
|
val = val.strip().upper()
|
||
|
|
for valid_val in valid_values:
|
||
|
|
if val == valid_val.upper():
|
||
|
|
return valid_val
|
||
|
|
return None
|
||
|
|
|
||
|
|
def clean_columns_combined(df, column_name, valid_values, new_column_name):
|
||
|
|
""" Cleans a column by applying an exact match check and updates it to a new column. """
|
||
|
|
changes = {}
|
||
|
|
original_values = df[column_name].unique()
|
||
|
|
|
||
|
|
def update_column(entry):
|
||
|
|
if pd.notna(entry):
|
||
|
|
terms = sanitize_value(entry).split(',')
|
||
|
|
for term in terms:
|
||
|
|
match = exact_match(term, valid_values)
|
||
|
|
if match:
|
||
|
|
return match
|
||
|
|
return None
|
||
|
|
|
||
|
|
df[new_column_name] = df[column_name].apply(update_column)
|
||
|
|
cleaned_values = df[new_column_name].unique()
|
||
|
|
return original_values, cleaned_values, changes
|
||
|
|
|
||
|
|
def get_closest_match(val, valid_values, similarity_threshold=0.5):
|
||
|
|
if pd.isna(val):
|
||
|
|
return None
|
||
|
|
val = val.strip().upper()
|
||
|
|
matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold)
|
||
|
|
return matches[0] if matches else None
|
||
|
|
|
||
|
|
def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
|
||
|
|
""" Applies fuzzy matching to a column in the dataframe and logs changes. """
|
||
|
|
changes = {}
|
||
|
|
original_values = df[column_name].unique()
|
||
|
|
|
||
|
|
def log_and_clean(entry):
|
||
|
|
if pd.notna(entry):
|
||
|
|
words = sanitize_value(entry).split(',')
|
||
|
|
cleaned_words = []
|
||
|
|
for word in words:
|
||
|
|
cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
|
||
|
|
if cleaned_word and word.strip().upper() != cleaned_word:
|
||
|
|
changes[word.strip()] = cleaned_word
|
||
|
|
cleaned_words.append(cleaned_word if cleaned_word else word.strip())
|
||
|
|
return ', '.join(cleaned_words)
|
||
|
|
return None
|
||
|
|
|
||
|
|
df[column_name] = df[column_name].apply(log_and_clean)
|
||
|
|
cleaned_values = df[column_name].unique()
|
||
|
|
return original_values, cleaned_values, changes
|
||
|
|
|
||
|
|
def extract_page_number(page_text):
|
||
|
|
match = re.search(r'Pages\s+(\d+)-\d+', page_text)
|
||
|
|
if match:
|
||
|
|
return match.group(1)
|
||
|
|
else:
|
||
|
|
return page_text
|
||
|
|
|
||
|
|
def clean_pagenumbers(df):
|
||
|
|
df['Page'] = df['Page'].apply(extract_page_number)
|
||
|
|
return df
|
||
|
|
|
||
|
|
def consolidate_csv(df, output_csv='output/consolidated.csv'):
|
||
|
|
df['page_num'] = df['Page'].apply(lambda x: x.split()[1])
|
||
|
|
|
||
|
|
grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({
|
||
|
|
'PRODUCT': lambda x: ', '.join(x.dropna().unique()),
|
||
|
|
'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()),
|
||
|
|
'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()),
|
||
|
|
'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()),
|
||
|
|
'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()),
|
||
|
|
'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()),
|
||
|
|
'page_num' : lambda x : ', '.join(x.dropna().unique())
|
||
|
|
}).reset_index()
|
||
|
|
|
||
|
|
grouped.to_csv(output_csv, index=False)
|
||
|
|
print(f"Consolidated CSV has been saved to {output_csv}")
|