doczyai-pipelines/src/qc_qa/confidence/summary.py

"""Batch-level confidence reporting.

After all per-file 1:1 scoring is done, this module produces two CSVs:

  - CONFIDENCE-SUMMARY.csv : per-field distribution stats
      columns: field, n, mean, p10, p50, p90, n_below_threshold,
               threshold, pct_below_threshold
  - CONFIDENCE-FLAGGED.csv  : per-file rows where any field's confidence
                              landed below the threshold (lowest first)
      columns: FILE_NAME, field, confidence, value, threshold

Both are derived from the per-row <FIELD>_CONF columns the rule scorer
writes into the 1:1 results.

Thresholds are read from config (default 0.6) and can be tuned per run
via the confidence_threshold CLI arg.
"""

from __future__ import annotations

import logging
from typing import Iterable, List, Tuple

import numpy as np
import pandas as pd

logger = logging.getLogger(__name__)

CONF_SUFFIX = "_CONF"
DEFAULT_THRESHOLD = 0.6


def _conf_columns(df: pd.DataFrame) -> List[str]:
    return [c for c in df.columns if c.endswith(CONF_SUFFIX) and c != CONF_SUFFIX]


def _field_name_from_conf(col: str) -> str:
    """Strip the _CONF suffix to get the underlying field name."""
    return col[: -len(CONF_SUFFIX)]


def _percentile(series: pd.Series, q: float) -> float:
    """np.nanpercentile, but defensive against an all-NaN series."""
    arr = pd.to_numeric(series, errors="coerce").to_numpy()
    arr = arr[~np.isnan(arr)]
    if arr.size == 0:
        return float("nan")
    return float(np.percentile(arr, q))


def compute_summary(
    final_df: pd.DataFrame, threshold: float = DEFAULT_THRESHOLD
) -> pd.DataFrame:
    """Build CONFIDENCE-SUMMARY.csv from the final results dataframe.

    Returns an empty DataFrame (correct columns) if there are no _CONF
    columns to summarize — callers can skip writing rather than crashing.
    """
    cols = [
        "field",
        "n",
        "mean",
        "p10",
        "p50",
        "p90",
        "n_below_threshold",
        "threshold",
        "pct_below_threshold",
    ]
    if final_df is None or final_df.empty:
        return pd.DataFrame(columns=cols)

    conf_cols = _conf_columns(final_df)
    if not conf_cols:
        return pd.DataFrame(columns=cols)

    rows = []
    for col in conf_cols:
        series = pd.to_numeric(final_df[col], errors="coerce").dropna()
        n = int(series.shape[0])
        if n == 0:
            rows.append(
                {
                    "field": _field_name_from_conf(col),
                    "n": 0,
                    "mean": float("nan"),
                    "p10": float("nan"),
                    "p50": float("nan"),
                    "p90": float("nan"),
                    "n_below_threshold": 0,
                    "threshold": threshold,
                    "pct_below_threshold": float("nan"),
                }
            )
            continue
        n_below = int((series < threshold).sum())
        rows.append(
            {
                "field": _field_name_from_conf(col),
                "n": n,
                "mean": round(float(series.mean()), 4),
                "p10": round(_percentile(series, 10), 4),
                "p50": round(_percentile(series, 50), 4),
                "p90": round(_percentile(series, 90), 4),
                "n_below_threshold": n_below,
                "threshold": threshold,
                "pct_below_threshold": round(100.0 * n_below / n, 2),
            }
        )

    out = pd.DataFrame(rows, columns=cols)
    out = out.sort_values(by="pct_below_threshold", ascending=False).reset_index(
        drop=True
    )
    return out


def compute_flagged(
    final_df: pd.DataFrame,
    threshold: float = DEFAULT_THRESHOLD,
    file_name_column: str = "FILE_NAME",
) -> pd.DataFrame:
    """Emit (FILE_NAME, field, confidence, value, threshold) rows for every
    cell below the threshold. Reviewer hit list.
    """
    cols = ["FILE_NAME", "field", "confidence", "value", "threshold"]
    if final_df is None or final_df.empty:
        return pd.DataFrame(columns=cols)

    conf_cols = _conf_columns(final_df)
    if not conf_cols:
        return pd.DataFrame(columns=cols)

    fn_col = file_name_column if file_name_column in final_df.columns else None

    flagged: List[dict] = []
    for col in conf_cols:
        field = _field_name_from_conf(col)
        series = pd.to_numeric(final_df[col], errors="coerce")
        below_mask = series < threshold
        if not below_mask.any():
            continue
        below = final_df.loc[below_mask, [c for c in (fn_col, field) if c]]
        for idx, row in below.iterrows():
            flagged.append(
                {
                    "FILE_NAME": row[fn_col] if fn_col else "",
                    "field": field,
                    "confidence": round(float(series.loc[idx]), 4),
                    "value": "" if field not in below.columns else row.get(field, ""),
                    "threshold": threshold,
                }
            )

    if not flagged:
        return pd.DataFrame(columns=cols)
    out = pd.DataFrame(flagged, columns=cols)
    out = out.sort_values(by="confidence", ascending=True).reset_index(drop=True)
    return out