jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Strip out dynamic postprocessing

Browse Source
This commit is contained in:
Katon Minhas
2026-02-03 01:38:49 -05:00
parent 03fda6013b
commit e8e8f1ba5a
5 changed files with 1 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/codes/code_funcs.py
+1 -26
View File
@@ -760,30 +760,6 @@ def extract_codes_from_service(answer_dict, constants: Constants):
return normalize_answer_dict_codes(answer_dict)
def fill_claim_type(answer_dicts):
"""
Fills in the AARETE_DERIVED_CLAIM_TYPE_CD field in each answer_dict with the mode of the existing values.
If the field is already populated, it remains unchanged.
Args:
answer_dicts (list[dict]): List of dictionaries containing 1:1 and 1:N fields (after merge process)
Returns:
list[dict]: List of dictionaries with AARETE_DERIVED_CLAIM_TYPE
"""
claim_types = [
d.get("AARETE_DERIVED_CLAIM_TYPE_CD")
for d in answer_dicts
if d.get("AARETE_DERIVED_CLAIM_TYPE_CD")
]
if not claim_types:
return answer_dicts
mode_claim_type = max(set(claim_types), key=claim_types.count)
for answer_dict in answer_dicts:
if not answer_dict.get("AARETE_DERIVED_CLAIM_TYPE_CD"):
answer_dict["AARETE_DERIVED_CLAIM_TYPE_CD"] = mode_claim_type
return answer_dicts
def code_breakout(merged_results: pd.DataFrame, constants: Constants):
"""
Processes a list of answer dictionaries to extract and fill in code-related information.
@@ -796,8 +772,7 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants):
"""
import concurrent.futures
# Fill empty AARETE_DERIVED_CLAIM_TYPE_CD with mode
answer_dicts = fill_claim_type(merged_results.to_dict(orient="records"))
answer_dicts = merged_results.to_dict(orient="records")
# Parallelize code extraction
def process_single_code(answer_dict):
src/pipelines/shared/postprocessing/postprocess.py
-2
View File
@@ -82,8 +82,6 @@ def postprocess(df, constants: Constants):
df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df)
df = postprocessing_funcs.fill_empty_dynamic(df)
df = postprocessing_funcs.attach_sid_column(df)
# Standardize output column order - this should ALWAYS be the final postprocessing step
src/pipelines/shared/postprocessing/postprocessing_funcs.py
-110
View File
@@ -764,116 +764,6 @@ def update_grouper_base_rate_and_grouper_pct_rate(df):
return df
def fill_empty_dynamic(df):
"""
For specified columns, fills NA values with the common value from the same EXHIBIT_PAGE group.
If all rows with the same EXHIBIT_PAGE value have only one unique non-NA value for a column,
this function fills any NA values in that column with that unique value.
Args:
df (pd.DataFrame): DataFrame with at least an EXHIBIT_PAGE column
Returns:
pd.DataFrame: DataFrame with filled values
"""
if "EXHIBIT_PAGE" not in df.columns or df.empty:
return df
# List of columns to check for filling
columns_to_fill = [
"AARETE_DERIVED_LOB",
"AARETE_DERIVED_PROGRAM",
"AARETE_DERIVED_PRODUCT",
"AARETE_DERIVED_NETWORK",
]
# Only process columns that actually exist in the dataframe
columns_to_fill = [col for col in columns_to_fill if col in df.columns]
# Make a copy to avoid SettingWithCopyWarning
result_df = df.copy()
# Check if AARETE_DERIVED_LOB column exists
has_lob_column = "AARETE_DERIVED_LOB" in result_df.columns
# First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only
if has_lob_column:
for (file_name, exhibit_page), group in result_df.groupby(
["FILE_NAME", "EXHIBIT_PAGE"]
):
# Get non-NA LOB values
column = "AARETE_DERIVED_LOB"
non_na_values = [
val for val in group[column].unique() if not string_utils.is_empty(val)
]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (
(result_df["FILE_NAME"] == file_name)
& (result_df["EXHIBIT_PAGE"] == exhibit_page)
& result_df[column].apply(string_utils.is_empty)
)
result_df.loc[mask, column] = fill_value
# Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB (if it exists)
if has_lob_column:
for (file_name, exhibit_page, lob), group in result_df.groupby(
["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]
):
for column in columns_to_fill:
# Skip AARETE_DERIVED_LOB as we already processed it
if column == "AARETE_DERIVED_LOB":
continue
# Get non-NA values
non_na_values = [
val
for val in group[column].unique()
if not string_utils.is_empty(val)
]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
mask = (
(result_df["FILE_NAME"] == file_name)
& (result_df["EXHIBIT_PAGE"] == exhibit_page)
& (result_df["AARETE_DERIVED_LOB"] == lob)
& result_df[column].apply(string_utils.is_empty)
)
result_df.loc[mask, column] = fill_value
else:
# No LOB column, just group by FILE_NAME and EXHIBIT_PAGE
for (file_name, exhibit_page), group in result_df.groupby(
["FILE_NAME", "EXHIBIT_PAGE"]
):
for column in columns_to_fill:
# Get non-NA values
non_na_values = [
val
for val in group[column].unique()
if not string_utils.is_empty(val)
]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (
(result_df["FILE_NAME"] == file_name)
& (result_df["EXHIBIT_PAGE"] == exhibit_page)
& result_df[column].apply(string_utils.is_empty)
)
result_df.loc[mask, column] = fill_value
return result_df
def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
"""
Ensures the given AARETE_DERIVED_SID columns exist in the DataFrame, fills missing values,
src/tests/test_code_funcs.py
-17
View File
@@ -399,23 +399,6 @@ class TestCodeFuncs(unittest.TestCase):
)
self.assertEqual(result["CODE_METHODOLOGY"], "Generic - No Match")
def test_fill_claim_type(self):
"""Test the fill_claim_type function for determining claim types."""
# Test with existing claim types
answer_dicts = [
{"AARETE_DERIVED_CLAIM_TYPE_CD": "H"},
{"AARETE_DERIVED_CLAIM_TYPE_CD": "H"},
{"AARETE_DERIVED_CLAIM_TYPE_CD": "M"},
{},
]
result = code_funcs.fill_claim_type(answer_dicts)
self.assertEqual(result[3]["AARETE_DERIVED_CLAIM_TYPE_CD"], "H")
# Test with all empty claim types
answer_dicts = [{}, {}]
result = code_funcs.fill_claim_type(answer_dicts)
self.assertEqual(result, answer_dicts)
@patch("src.codes.code_funcs.fill_claim_type")
@patch("src.codes.code_funcs.extract_codes_from_service")
def test_code_breakout(self, mock_extract_codes, mock_fill_claim_type):
src/tests/test_postprocess.py
-162
View File
@@ -15,7 +15,6 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import (
remove_redundant_reimb_info,
rename_columns,
validate_and_reformat_date,
fill_empty_dynamic,
)
@@ -433,167 +432,6 @@ class TestPostprocessFunctions(unittest.TestCase):
result_df7 = deduplicate_provider_columns(input_df7)
pd.testing.assert_frame_equal(result_df7, input_df7)
def test_fill_empty_dynamic(self):
"""Tests the fill_empty_dynamic function that fills NA values with common values from the same EXHIBIT_PAGE group.
Tests:
1. Basic filling - fills NA values with the common value for the same EXHIBIT_PAGE
2. Multiple columns - correctly fills multiple columns independently
3. Multiple file/page combinations - respects FILE_NAME and EXHIBIT_PAGE boundaries
4. No common value - doesn't fill when multiple non-NA values exist
5. All NA values - doesn't fill when all values are NA
6. Missing columns - returns unchanged DataFrame when key columns are missing
"""
# Test case 1: Basic filling for a single column
input_df1 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "", None],
}
)
expected_df1 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"],
}
)
result_df1 = fill_empty_dynamic(input_df1)
pd.testing.assert_frame_equal(result_df1, expected_df1)
# Test case 2: Multiple columns
input_df2 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "", None],
"AARETE_DERIVED_PRODUCT": ["Product A", None, ""],
}
)
expected_df2 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"],
"AARETE_DERIVED_PRODUCT": ["Product A", "Product A", "Product A"],
}
)
result_df2 = fill_empty_dynamic(input_df2)
pd.testing.assert_frame_equal(result_df2, expected_df2)
# Test case 3: Multiple file/page combinations
input_df3 = pd.DataFrame(
{
"FILE_NAME": [
"file1.pdf",
"file1.pdf",
"file1.pdf",
"file1.pdf",
"file2.pdf",
"file2.pdf",
],
"EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [
"Commercial",
"",
"Medicare",
None,
"Medicaid",
"",
],
}
)
expected_df3 = pd.DataFrame(
{
"FILE_NAME": [
"file1.pdf",
"file1.pdf",
"file1.pdf",
"file1.pdf",
"file2.pdf",
"file2.pdf",
],
"EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [
"Commercial",
"Commercial",
"Medicare",
"Medicare",
"Medicaid",
"Medicaid",
],
}
)
result_df3 = fill_empty_dynamic(input_df3)
pd.testing.assert_frame_equal(result_df3, expected_df3)
# Test case 4: No common value (multiple non-NA values exist)
input_df4 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Medicare", None],
}
)
expected_df4 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [
"Commercial",
"Medicare",
None,
], # Should remain unchanged
}
)
result_df4 = fill_empty_dynamic(input_df4)
pd.testing.assert_frame_equal(result_df4, expected_df4)
# Test case 5: All NA values
input_df5 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [None, "", None],
}
)
expected_df5 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [None, "", None], # Should remain unchanged
}
)
result_df5 = fill_empty_dynamic(input_df5)
pd.testing.assert_frame_equal(result_df5, expected_df5)
# Test case 6: Missing EXHIBIT_PAGE column
input_df6 = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf"],
"AARETE_DERIVED_LOB": ["Commercial", None],
}
)
result_df6 = fill_empty_dynamic(input_df6)
pd.testing.assert_frame_equal(result_df6, input_df6) # Should remain unchanged
# Test case 7: Empty DataFrame
empty_df = pd.DataFrame()
result_empty = fill_empty_dynamic(empty_df)
pd.testing.assert_frame_equal(result_empty, empty_df) # Should remain unchanged
if __name__ == "__main__":
unittest.main()