Merged in feature/attach_sid (pull request #747)

new functionality to generate SID

* new functionality to generate SID

* replacing xx with '' if new col created
* Merged main into feature/attach_sid


Approved-by: Katon Minhas

fieldExtraction/src/investment/postprocess.py

@@ -88,6 +88,8 @@ def postprocess(df, constants: Constants):
 
     df = postprocessing_funcs.fill_empty_dynamic(df)
 
+    df = postprocessing_funcs.attach_sid_column(df)
+
     # Standardize output column order - this should ALWAYS be the final postprocessing step
     df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
 

fieldExtraction/src/investment/postprocessing_funcs.py

@@ -869,3 +869,52 @@ def fill_empty_dynamic(df):
                 result_df.loc[mask, column] = fill_value
                 
     return result_df
+
+
+def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Ensures the given SID columns exist in the DataFrame, fills missing values,
+    and creates a unique 'SID' identifier column — while keeping all original columns.
+
+    Steps:
+    1. Verify that all columns listed in sid_cols exist in the DataFrame.
+       - If any are missing, create them and fill with ''.
+    2. Concatenate all sid_cols values with '__' as a separator to form 'SID'.
+    3. If any value in sid_cols is blank or NaN, replace it with 'xx' in the concatenation.
+    4. Add the 'SID' column to the DataFrame.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame containing the source columns.
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with all original columns plus a new 'SID' column.
+    """
+    sid_cols = [
+        'CLIENT_NAME',
+        'PAYER_NAME',
+        'PROV_GROUP_TIN',
+        'DOCUMENT_TYPE',
+        'AARETE_DERIVED_AMENDMENT_NUM',
+        'AARETE_DERIVED_CLAIM_TYPE_CD',
+        'AARETE_DERIVED_LOB',
+        'AARETE_DERIVED_EFFECTIVE_DT'
+    ]
+
+    # Make a copy to avoid mutating the original DataFrame
+    df = df.copy()
+
+    # Ensure all SID columns exist; create missing ones filled with 'xx'
+    for col in sid_cols:
+        if col not in df.columns:
+            df[col] = ''
+
+    # Create the concatenated SID column (replace blanks/NaN only here)
+    df['SID'] = (
+        df.assign(**{col: df[col].fillna('').astype(str) for col in sid_cols})
+          .apply(lambda row: '__'.join(val if val.strip() else 'xx' for val in row[sid_cols]), axis=1)
+    )
+
+    return df