jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/fill_claim_type (pull request #874)

Browse Source 
Bugfix/fill claim type

* Handle if AARETE_DERIVED_CLAIM_TYPE_CD comes in as list

* add unit tests

* Black format


Approved-by: Siddhant Medar
This commit is contained in:
Katon Minhas
2026-02-09 19:12:15 +00:00
parent fc178587c8
commit 47472c0f32
2 changed files with 219 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipelines/shared/postprocessing/postprocessing_funcs.py
+9 -2
View File
@@ -880,8 +880,15 @@ def fill_claim_type_from_title(df: pd.DataFrame) -> pd.DataFrame:
file_mask = df["FILE_NAME"] == file_name
file_rows = df.loc[file_mask, "AARETE_DERIVED_CLAIM_TYPE_CD"]
# Get non-empty values for this file
non_empty_values = [v for v in file_rows if not string_utils.is_empty(v)]
# Get non-empty values for this file, normalizing lists to strings
non_empty_values = []
for v in file_rows:
if not string_utils.is_empty(v):
# Convert lists to their first element if present
if isinstance(v, list) and len(v) > 0:
non_empty_values.append(v[0])
elif not isinstance(v, list):
non_empty_values.append(v)
if non_empty_values:
# Use mode (most common value) to fill empty rows in this file
src/tests/test_postprocess.py
+210
View File
@@ -5,6 +5,7 @@ import pandas as pd
import pytest
from src.pipelines.shared.postprocessing.postprocessing_funcs import (
deduplicate_provider_columns,
fill_claim_type_from_title,
flatten_singleton_string_list,
format_rate_fields_with_commas,
normalize_auto_renewal_term,
@@ -452,6 +453,215 @@ class TestPostprocessFunctions(unittest.TestCase):
result_df7 = deduplicate_provider_columns(input_df7)
pd.testing.assert_frame_equal(result_df7, input_df7)
def test_fill_claim_type_from_title_mode_fill(self):
"""Test filling empty values using mode from same file."""
input_df = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf", "file2.pdf"],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["M", "M", "", "H"],
"CONTRACT_TITLE": ["Test", "Test", "Test", "Hospital Agreement"],
}
)
result_df = fill_claim_type_from_title(input_df)
# Empty value in file1 should be filled with "M" (mode)
assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
# file2 value should remain unchanged
assert result_df.loc[3, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H"
def test_fill_claim_type_from_title_list_values(self):
"""Test handling list values (bug fix scenario)."""
input_df = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"AARETE_DERIVED_CLAIM_TYPE_CD": [["M"], ["M"], ""],
"CONTRACT_TITLE": ["Test", "Test", "Test"],
}
)
result_df = fill_claim_type_from_title(input_df)
# Should handle list values and fill empty row with mode
assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
def test_fill_claim_type_from_title_professional_keywords(self):
"""Test inferring M from professional keywords in CONTRACT_TITLE."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": [
"Physician Services Agreement",
"Professional Provider Agreement",
"Medical Group Contract",
"Participating Provider Agreement",
],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", "", ""],
}
)
result_df = fill_claim_type_from_title(input_df)
# All should be inferred as "M"
assert all(result_df["AARETE_DERIVED_CLAIM_TYPE_CD"] == "M")
def test_fill_claim_type_from_title_ancillary_keywords(self):
"""Test inferring M from ancillary keywords in CONTRACT_TITLE."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": [
"Home Health Services Agreement",
"DME Provider Contract",
"Laboratory Services Agreement",
"Behavioral Health Agreement",
"Ambulatory Surgical Center",
],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", "", "", ""],
}
)
result_df = fill_claim_type_from_title(input_df)
# All should be inferred as "M"
assert all(result_df["AARETE_DERIVED_CLAIM_TYPE_CD"] == "M")
def test_fill_claim_type_from_title_institutional_keywords(self):
"""Test inferring H from institutional keywords in CONTRACT_TITLE."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": [
"Hospital Services Agreement",
"Institutional Provider Contract",
"Facility Agreement",
"Inpatient Services",
],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", "", ""],
}
)
result_df = fill_claim_type_from_title(input_df)
# All should be inferred as "H"
assert all(result_df["AARETE_DERIVED_CLAIM_TYPE_CD"] == "H")
def test_fill_claim_type_from_title_case_insensitive(self):
"""Test that keyword matching is case-insensitive."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": [
"physician services agreement",
"HOSPITAL AGREEMENT",
"AnCiLlArY sErViCeS",
],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", ""],
}
)
result_df = fill_claim_type_from_title(input_df)
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H"
assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
def test_fill_claim_type_from_title_preserve_existing(self):
"""Test that existing non-empty values are preserved."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": ["Hospital Agreement", "Physician Agreement"],
"AARETE_DERIVED_CLAIM_TYPE_CD": [
"M",
"H",
], # Opposite of what keywords would infer
}
)
result_df = fill_claim_type_from_title(input_df)
# Should preserve existing values even if they don't match keywords
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H"
def test_fill_claim_type_from_title_no_match(self):
"""Test that no value is set when no keywords match."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": ["Generic Contract", "Some Agreement"],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["", ""],
}
)
result_df = fill_claim_type_from_title(input_df)
# Should remain empty when no keywords match
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == ""
assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == ""
def test_fill_claim_type_from_title_missing_column(self):
"""Test that function handles missing AARETE_DERIVED_CLAIM_TYPE_CD column."""
input_df = pd.DataFrame(
{
"CONTRACT_TITLE": ["Physician Agreement"],
}
)
result_df = fill_claim_type_from_title(input_df)
# Should create the column and infer value
assert "AARETE_DERIVED_CLAIM_TYPE_CD" in result_df.columns
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
def test_fill_claim_type_from_title_missing_contract_title(self):
"""Test that function handles missing CONTRACT_TITLE column."""
input_df = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf"],
"AARETE_DERIVED_CLAIM_TYPE_CD": [""],
}
)
result_df = fill_claim_type_from_title(input_df)
# Should return df unchanged (can't infer without CONTRACT_TITLE)
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == ""
def test_fill_claim_type_from_title_mixed_scenario(self):
"""Test combined mode filling and keyword inference."""
input_df = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf", "file2.pdf", "file2.pdf"],
"CONTRACT_TITLE": [
"Test",
"Test",
"Physician Agreement",
"Generic",
],
"AARETE_DERIVED_CLAIM_TYPE_CD": ["H", "", "", ""],
}
)
result_df = fill_claim_type_from_title(input_df)
# Row 1 should be filled with "H" (mode from file1)
assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H"
# Row 2 should be inferred as "M" from keyword
assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
# Row 3 should remain empty (no mode in file2, no keyword match)
assert result_df.loc[3, "AARETE_DERIVED_CLAIM_TYPE_CD"] == ""
def test_fill_claim_type_from_title_empty_list_values(self):
"""Test handling empty list values."""
input_df = pd.DataFrame(
{
"FILE_NAME": ["file1.pdf", "file1.pdf"],
"AARETE_DERIVED_CLAIM_TYPE_CD": [[], "M"],
"CONTRACT_TITLE": ["Physician Agreement", "Test"],
}
)
result_df = fill_claim_type_from_title(input_df)
# Empty list should be treated as empty and filled with mode or inferred
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
if __name__ == "__main__":
unittest.main()