Merged postprocessing updates

src/postprocessingfuncs.py

@@ -6,12 +6,11 @@ def sanitize_value(value):
     """ Remove brackets from list items and clean the values. """
     if pd.isna(value):
         return value
+    if isinstance(value, list):
+        value = ', '.join(str(v) for v in value)
     if isinstance(value, str):
         value = value.strip('[]')
-        if ',' in value:
-            value = ', '.join([item.strip(" '") for item in value.split(',')])
-        else:
-            value = value.strip(" '")
+        value = ', '.join([item.strip(" '") for item in value.split(',')])
     return value
 
 def exact_match(val, valid_values):
@@ -24,11 +23,12 @@ def exact_match(val, valid_values):
 def clean_columns_combined(df, column_name, valid_values, new_column_name):
     """ Cleans a column by applying an exact match check and updates it to a new column. """
     changes = {}
+    df[column_name] = df[column_name].apply(sanitize_value)
     original_values = df[column_name].unique()
 
     def update_column(entry):
         if pd.notna(entry):
-            terms = sanitize_value(entry).split(',')
+            terms = entry.split(',')
             for term in terms:
                 match = exact_match(term, valid_values)
                 if match:
@@ -39,7 +39,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name):
     cleaned_values = df[new_column_name].unique()
     return original_values, cleaned_values, changes
 
-def get_closest_match(val, valid_values, similarity_threshold=0.5):
+def get_closest_match(val, valid_values, similarity_threshold=0.7):
     if pd.isna(val):
         return None
     val = val.strip().upper()
@@ -49,11 +49,12 @@ def get_closest_match(val, valid_values, similarity_threshold=0.5):
 def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
     """ Applies fuzzy matching to a column in the dataframe and logs changes. """
     changes = {}
+    df[column_name] = df[column_name].apply(sanitize_value)
     original_values = df[column_name].unique()
     
     def log_and_clean(entry):
         if pd.notna(entry):
-            words = sanitize_value(entry).split(',')
+            words = entry.split(',')
             cleaned_words = []
             for word in words:
                 cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
@@ -75,21 +76,29 @@ def extract_page_number(page_text):
         return page_text
 
 def clean_pagenumbers(df):
-    df['Page'] = df['Page'].apply(extract_page_number)
+    df['page_num'] = df['page_num'].apply(extract_page_number)
     return df
 
-def consolidate_csv(df, output_csv='output/consolidated.csv'):
-    df['page_num'] = df['Page'].apply(lambda x: x.split()[1])
-
-    grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({
-        'PRODUCT': lambda x: ', '.join(x.dropna().unique()),
-        'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()),
-        'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()),
-        'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()),
-        'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()),
-        'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()),
-        'page_num' : lambda x : ', '.join(x.dropna().unique())
-    }).reset_index()
-
-    grouped.to_csv(output_csv, index=False)
-    print(f"Consolidated CSV has been saved to {output_csv}")
+def correct_misplaced_values(df, columns, valid_values_dict):
+    """
+    Check and correct misplaced values across specified columns.
+    """
+    for index, row in df.iterrows():
+        for col in columns:
+            if pd.notna(row[col]):
+                terms = row[col].split(',')
+                for term in terms:
+                    term = term.strip()
+                    for target_col, valid_values in valid_values_dict.items():
+                        if target_col != col:
+                            match = exact_match(term, valid_values)
+                            if match:
+                                if pd.isna(row[target_col]) or not row[target_col].strip():
+                                    df.at[index, target_col] = match
+                                    df.at[index, col] = None
+                                else:
+                                    current_value = row[target_col].strip()
+                                    if get_closest_match(match, [current_value], 0.8) is None:
+                                        df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell"
+                                        df.at[index, col] = None
+    return df