doczyai-pipelines/src/qc_qa/confidence/scorer.py

"""Combine rule-based signals into a per-field confidence score.

Inputs:
    field_name              : the HSC field name (used to pick FieldType + keywords)
    value                   : the LLM-extracted value
    metadata                : dict from prompt_hsc_single_field; carries the
                              retrieved-chunks text and the LLM-stated confidence
    contract_text           : full cleaned contract text (for cross-checking)

Output:
    final score in [0.0, 1.0], rounded to 4 decimals.

Method:
    1. Resolve FieldType -> weight preset.
    2. Compute each signal that has non-zero weight for that type.
       Skip signals that return None (not applicable for this value).
    3. Renormalize weights over the signals that fired.
    4. Weighted average -> rule_score in [0, 1].
    5. Blend lightly with the LLM-stated confidence as a smoothing prior.
       (T3 will replace this prior with a calibrated LLM verifier.)
"""

from __future__ import annotations

import logging
import re
from typing import Optional

from src.qc_qa.confidence import calibration as cal
from src.qc_qa.confidence import field_types as ft
from src.qc_qa.confidence import signals as sig


# How much weight to give the LLM-stated confidence in the final blend.
# Low on purpose: the rules dominate. T3 will replace this with a calibrated
# verifier.
_LLM_PRIOR_WEIGHT = 0.15

logger = logging.getLogger(__name__)


def _coerce_float(value: object) -> Optional[float]:
    if value is None:
        return None
    try:
        out = float(value)
    except (TypeError, ValueError):
        return None
    if out != out:  # NaN
        return None
    return max(0.0, min(1.0, out))


_STOPWORDS = frozenset(
    {
        # Interrogatives / connectives we keep filtering out.
        "what",
        "where",
        "when",
        "which",
        "whose",
        "there",
        "their",
        "these",
        "those",
        "about",
        "above",
        "below",
        "after",
        "before",
        "again",
        "further",
        "between",
        "during",
        "until",
        "while",
        "could",
        "would",
        "should",
        "shall",
        "might",
        # Common short stopwords that slip past length >= 4 but carry no
        # domain meaning, so they shouldn't contribute to na_check hits.
        "this",
        "that",
        "with",
        "from",
        "into",
        "have",
        "your",
        "more",
        "only",
        "such",
        "than",
        "they",
        "them",
        "also",
        "each",
        "very",
        "some",
        "most",
        "many",
        "much",
        "into",
        "been",
        "were",
        "will",
        "must",
        # Contract-boilerplate vocabulary. These appear in nearly every
        # healthcare contract regardless of which field is being scored,
        # so they're useless as na_check evidence and would otherwise
        # flag every doc as suspicious.
        "contract",
        "contracts",
        "contractual",
        "agreement",
        "agreements",
        "document",
        "documents",
        "provider",
        "providers",
        "payer",
        "payers",
        "party",
        "parties",
        "service",
        "services",
        "section",
        "sections",
        "article",
        "articles",
        "paragraph",
        "term",
        "terms",
        "covered",
        # Prompt-instruction noise (these talk about the prompt, not the
        # contract, so they're not useful evidence either way).
        "value",
        "field",
        "given",
        "named",
        "answer",
        "extract",
        "extracted",
        "return",
        "follow",
        "rules",
        "guidelines",
        "instructions",
        "letters",
        "numeric",
        "alphanumeric",
        "include",
        "including",
        "format",
        "string",
        "below",
    }
)

_KEYWORD_RE = re.compile(r"[A-Za-z][A-Za-z\-]{3,}")


def _field_keywords(field_name: str, field_prompt: str = "") -> list[str]:
    """Keyword set for the na_check signal.

    Uses the field name as the keyword source. Field names like
    CONTRACT_AMENDMENT_NUM are terse and domain-specific by design --
    split on underscores, drop boilerplate ("contract"), drop short
    tokens ("num"), and what's left is the actual concept ("amendment").
    That's a far cleaner anti-hallucination signal than scanning a
    verbose prompt that drags in generic English ("written", "letters",
    "title", "numeric") that appears in every contract regardless.

    The prompt is a TRUE fallback: only consulted when the field name
    yields nothing useful. For every HSC field today, the field name
    produces at least one keyword, so the prompt never runs in practice.
    Caps the result at 25 keywords.
    """
    seen: dict[str, None] = {}

    def _add(token: str) -> bool:
        """Add a normalized token; return True when the cap is reached."""
        tok = token.lower()
        if len(tok) < 4 or tok in _STOPWORDS or tok in seen:
            return False
        seen[tok] = None
        return len(seen) >= 25

    # 1. Field name -- preferred source.
    if field_name:
        for tok in re.split(r"[_\W]+", field_name):
            if _add(tok):
                return list(seen.keys())

    # 2. Prompt -- only as fallback when the field name had nothing
    #    useful to contribute (rare in practice).
    if not seen and field_prompt:
        for tok in _KEYWORD_RE.findall(field_prompt):
            if _add(tok):
                return list(seen.keys())

    return list(seen.keys())


def score_field(
    field_name: str,
    value: object,
    metadata: dict | None,
    contract_text: str = "",
    field_prompt: str = "",
) -> float:
    """Compute the rule-based confidence score for a single field.

    Always returns a finite float in [0.0, 1.0]. On any unexpected error,
    falls back to the LLM-stated confidence (or 0.5) and logs a warning --
    a bad rule score must never crash the pipeline.
    """
    metadata = metadata or {}
    try:
        field_type = ft.get_field_type(field_name)
        weights = ft.get_weight_preset(field_type)

        retrieved_text = str(metadata.get("retrieved_chunks_text") or "")
        keywords = _field_keywords(field_name, field_prompt)

        scores: dict[str, float] = {}

        if weights.get(ft.EXACT, 0) > 0:
            v = sig.signal_exact(value, contract_text or retrieved_text)
            if v is not None:
                scores[ft.EXACT] = v

        if weights.get(ft.TOKEN_OVERLAP, 0) > 0:
            v = sig.signal_token_overlap(value, contract_text or retrieved_text)
            if v is not None:
                scores[ft.TOKEN_OVERLAP] = v

        if weights.get(ft.FUZZY, 0) > 0:
            # Fuzzy is O(n*m); keep it scoped to retrieved chunks when present.
            haystack = retrieved_text or contract_text
            v = sig.signal_fuzzy(value, haystack)
            if v is not None:
                scores[ft.FUZZY] = v

        if weights.get(ft.NUMBER_MATCH, 0) > 0:
            v = sig.signal_number_match(value, contract_text or retrieved_text)
            if v is not None:
                scores[ft.NUMBER_MATCH] = v

        if weights.get(ft.REGEX, 0) > 0:
            v = sig.signal_regex(value, field_type)
            if v is not None:
                scores[ft.REGEX] = v

        if weights.get(ft.GROUNDING, 0) > 0:
            v = sig.signal_grounding(value, retrieved_text)
            if v is not None:
                scores[ft.GROUNDING] = v

        if weights.get(ft.NA_CHECK, 0) > 0:
            v = sig.signal_na_check(value, contract_text, keywords)
            if v is not None:
                scores[ft.NA_CHECK] = v

        if not scores:
            # No signal could fire -- fall back to the LLM's self-reported
            # confidence (which we normalize) or a neutral 0.5.
            return _coerce_float(metadata.get("confidence")) or 0.5

        # Renormalize weights over the signals that actually fired.
        total_weight = sum(weights[name] for name in scores)
        if total_weight <= 0:
            return _coerce_float(metadata.get("confidence")) or 0.5
        rule_score = sum(scores[n] * weights[n] for n in scores) / total_weight

        # Light blend with the LLM-stated confidence as a prior. The
        # rule-based scorer dominates; the prior just smooths very-low
        # rule scores when the LLM was very confident (and vice versa).
        llm_conf = _coerce_float(metadata.get("confidence"))
        if llm_conf is None:
            final = rule_score
        else:
            final = (1 - _LLM_PRIOR_WEIGHT) * rule_score + _LLM_PRIOR_WEIGHT * llm_conf

        # Map raw -> empirically-calibrated confidence using the
        # per-field-type curve. Identity by default; becomes meaningful
        # once the curve is backfilled from QC data.
        final = cal.calibrate(field_type, final)

        return round(max(0.0, min(1.0, final)), 4)

    except Exception as exc:  # pragma: no cover - defensive
        logger.warning(
            "score_field failed for %s; falling back to LLM-stated conf: %s",
            field_name,
            exc,
        )
        return _coerce_float((metadata or {}).get("confidence")) or 0.5