From 384f0444f7f214e9ff490ad4ba885383f58be36c Mon Sep 17 00:00:00 2001 From: Faizan Mohiuddin Date: Wed, 13 May 2026 14:10:19 +0000 Subject: [PATCH] Merged in bugfix/confidence-flagged-missing-field-col (pull request #1006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix KeyError in compute_flagged when a *_CONF column has no value sibling * Fix KeyError in compute_flagged when a *_CONF column has no value sibling Production hit a hard crash at the end of every run: KeyError: "['DYNAMIC_PRIMARY_ENTITIES'] not in index" src/qc_qa/confidence/summary.py:143 compute_flagged was iterating over *_CONF columns and unconditionally indexing the dataframe with both the FILE_NAME column and the stripped value column. That assumed every _CONF column has a sibling value column in final_df. That isn't always true: dynamic-primary features carry only the _CONF side (their value side is dropped by reorder_columns since it isn't in FIELD_FORMAT_MAPPING but its _CONF suffix matches the explicit _CONF carve-out). When the model produced a below-threshold score for one of these and the value column was absent, pandas .loc raised KeyError and the runner crashed. Fix: - Build the .loc column list defensively: drop any name that isn't actually present in final_df. - Make the per-row dict construction robust to either FILE_NAME or … Approved-by: Katon Minhas --- src/qc_qa/confidence/summary.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/qc_qa/confidence/summary.py b/src/qc_qa/confidence/summary.py index 72d1f7f..2c65d44 100644 --- a/src/qc_qa/confidence/summary.py +++ b/src/qc_qa/confidence/summary.py @@ -140,14 +140,22 @@ def compute_flagged( below_mask = series < threshold if not below_mask.any(): continue - below = final_df.loc[below_mask, [c for c in (fn_col, field) if c]] + # Only select columns that actually exist. A _CONF column can + # survive into final_df without its sibling value column -- + # e.g. an internal-only confidence-bearing column whose value side + # was dropped by reorder_columns. Without this guard, .loc throws + # KeyError: "[''] not in index" and crashes the run. + select_cols = [c for c in (fn_col, field) if c and c in final_df.columns] + below = final_df.loc[below_mask, select_cols] for idx, row in below.iterrows(): flagged.append( { - "FILE_NAME": row[fn_col] if fn_col else "", + "FILE_NAME": ( + row[fn_col] if fn_col and fn_col in below.columns else "" + ), "field": field, "confidence": round(float(series.loc[idx]), 4), - "value": "" if field not in below.columns else row.get(field, ""), + "value": row.get(field, "") if field in below.columns else "", "threshold": threshold, } )