diff --git a/fieldExtraction/src/investment/postprocess.py b/fieldExtraction/src/investment/postprocess.py index 85e9b0b..284fe43 100644 --- a/fieldExtraction/src/investment/postprocess.py +++ b/fieldExtraction/src/investment/postprocess.py @@ -88,6 +88,8 @@ def postprocess(df, constants: Constants): df = postprocessing_funcs.fill_empty_dynamic(df) + df = postprocessing_funcs.attach_sid_column(df) + # Standardize output column order - this should ALWAYS be the final postprocessing step df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) diff --git a/fieldExtraction/src/investment/postprocessing_funcs.py b/fieldExtraction/src/investment/postprocessing_funcs.py index b717ec0..ef77a48 100644 --- a/fieldExtraction/src/investment/postprocessing_funcs.py +++ b/fieldExtraction/src/investment/postprocessing_funcs.py @@ -869,3 +869,52 @@ def fill_empty_dynamic(df): result_df.loc[mask, column] = fill_value return result_df + + +def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame: + """ + Ensures the given SID columns exist in the DataFrame, fills missing values, + and creates a unique 'SID' identifier column — while keeping all original columns. + + Steps: + 1. Verify that all columns listed in sid_cols exist in the DataFrame. + - If any are missing, create them and fill with ''. + 2. Concatenate all sid_cols values with '__' as a separator to form 'SID'. + 3. If any value in sid_cols is blank or NaN, replace it with 'xx' in the concatenation. + 4. Add the 'SID' column to the DataFrame. + + Parameters + ---------- + df : pd.DataFrame + Input DataFrame containing the source columns. + Returns + ------- + pd.DataFrame + DataFrame with all original columns plus a new 'SID' column. + """ + sid_cols = [ + 'CLIENT_NAME', + 'PAYER_NAME', + 'PROV_GROUP_TIN', + 'DOCUMENT_TYPE', + 'AARETE_DERIVED_AMENDMENT_NUM', + 'AARETE_DERIVED_CLAIM_TYPE_CD', + 'AARETE_DERIVED_LOB', + 'AARETE_DERIVED_EFFECTIVE_DT' + ] + + # Make a copy to avoid mutating the original DataFrame + df = df.copy() + + # Ensure all SID columns exist; create missing ones filled with 'xx' + for col in sid_cols: + if col not in df.columns: + df[col] = '' + + # Create the concatenated SID column (replace blanks/NaN only here) + df['SID'] = ( + df.assign(**{col: df[col].fillna('').astype(str) for col in sid_cols}) + .apply(lambda row: '__'.join(val if val.strip() else 'xx' for val in row[sid_cols]), axis=1) + ) + + return df