jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Strip out dynamic postprocessing

Browse Source
This commit is contained in:
Katon Minhas
2026-02-03 01:38:49 -05:00
parent 03fda6013b
commit e8e8f1ba5a
5 changed files with 1 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipelines/shared/postprocessing/postprocessing_funcs.py
-110
View File
@@ -764,116 +764,6 @@ def update_grouper_base_rate_and_grouper_pct_rate(df):
return df
def fill_empty_dynamic(df):
"""
For specified columns, fills NA values with the common value from the same EXHIBIT_PAGE group.
If all rows with the same EXHIBIT_PAGE value have only one unique non-NA value for a column,
this function fills any NA values in that column with that unique value.
Args:
df (pd.DataFrame): DataFrame with at least an EXHIBIT_PAGE column
Returns:
pd.DataFrame: DataFrame with filled values
"""
if "EXHIBIT_PAGE" not in df.columns or df.empty:
return df
# List of columns to check for filling
columns_to_fill = [
"AARETE_DERIVED_LOB",
"AARETE_DERIVED_PROGRAM",
"AARETE_DERIVED_PRODUCT",
"AARETE_DERIVED_NETWORK",
]
# Only process columns that actually exist in the dataframe
columns_to_fill = [col for col in columns_to_fill if col in df.columns]
# Make a copy to avoid SettingWithCopyWarning
result_df = df.copy()
# Check if AARETE_DERIVED_LOB column exists
has_lob_column = "AARETE_DERIVED_LOB" in result_df.columns
# First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only
if has_lob_column:
for (file_name, exhibit_page), group in result_df.groupby(
["FILE_NAME", "EXHIBIT_PAGE"]
):
# Get non-NA LOB values
column = "AARETE_DERIVED_LOB"
non_na_values = [
val for val in group[column].unique() if not string_utils.is_empty(val)
]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (
(result_df["FILE_NAME"] == file_name)
& (result_df["EXHIBIT_PAGE"] == exhibit_page)
& result_df[column].apply(string_utils.is_empty)
)
result_df.loc[mask, column] = fill_value
# Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB (if it exists)
if has_lob_column:
for (file_name, exhibit_page, lob), group in result_df.groupby(
["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]
):
for column in columns_to_fill:
# Skip AARETE_DERIVED_LOB as we already processed it
if column == "AARETE_DERIVED_LOB":
continue
# Get non-NA values
non_na_values = [
val
for val in group[column].unique()
if not string_utils.is_empty(val)
]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
mask = (
(result_df["FILE_NAME"] == file_name)
& (result_df["EXHIBIT_PAGE"] == exhibit_page)
& (result_df["AARETE_DERIVED_LOB"] == lob)
& result_df[column].apply(string_utils.is_empty)
)
result_df.loc[mask, column] = fill_value
else:
# No LOB column, just group by FILE_NAME and EXHIBIT_PAGE
for (file_name, exhibit_page), group in result_df.groupby(
["FILE_NAME", "EXHIBIT_PAGE"]
):
for column in columns_to_fill:
# Get non-NA values
non_na_values = [
val
for val in group[column].unique()
if not string_utils.is_empty(val)
]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (
(result_df["FILE_NAME"] == file_name)
& (result_df["EXHIBIT_PAGE"] == exhibit_page)
& result_df[column].apply(string_utils.is_empty)
)
result_df.loc[mask, column] = fill_value
return result_df
def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
"""
Ensures the given AARETE_DERIVED_SID columns exist in the DataFrame, fills missing values,