300 lines
9.2 KiB
Python
300 lines
9.2 KiB
Python
|
"""Combine rule-based signals into a per-field confidence score.
|
||
|
|
|
||
|
|
Inputs:
|
||
|
|
field_name : the HSC field name (used to pick FieldType + keywords)
|
||
|
|
value : the LLM-extracted value
|
||
|
|
metadata : dict from prompt_hsc_single_field; carries the
|
||
|
|
retrieved-chunks text and the LLM-stated confidence
|
||
|
|
contract_text : full cleaned contract text (for cross-checking)
|
||
|
|
|
||
|
|
Output:
|
||
|
|
final score in [0.0, 1.0], rounded to 4 decimals.
|
||
|
|
|
||
|
|
Method:
|
||
|
|
1. Resolve FieldType -> weight preset.
|
||
|
|
2. Compute each signal that has non-zero weight for that type.
|
||
|
|
Skip signals that return None (not applicable for this value).
|
||
|
|
3. Renormalize weights over the signals that fired.
|
||
|
|
4. Weighted average -> rule_score in [0, 1].
|
||
|
|
5. Blend lightly with the LLM-stated confidence as a smoothing prior.
|
||
|
|
(T3 will replace this prior with a calibrated LLM verifier.)
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import re
|
||
|
|
from typing import Optional
|
||
|
|
|
||
|
|
from src.qc_qa.confidence import calibration as cal
|
||
|
|
from src.qc_qa.confidence import field_types as ft
|
||
|
|
from src.qc_qa.confidence import signals as sig
|
||
|
|
|
||
|
|
|
||
|
|
# How much weight to give the LLM-stated confidence in the final blend.
|
||
|
|
# Low on purpose: the rules dominate. T3 will replace this with a calibrated
|
||
|
|
# verifier.
|
||
|
|
_LLM_PRIOR_WEIGHT = 0.15
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
def _coerce_float(value: object) -> Optional[float]:
|
||
|
|
if value is None:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
out = float(value)
|
||
|
|
except (TypeError, ValueError):
|
||
|
|
return None
|
||
|
|
if out != out: # NaN
|
||
|
|
return None
|
||
|
|
return max(0.0, min(1.0, out))
|
||
|
|
|
||
|
|
|
||
|
|
_STOPWORDS = frozenset(
|
||
|
|
{
|
||
|
|
# Interrogatives / connectives we keep filtering out.
|
||
|
|
"what",
|
||
|
|
"where",
|
||
|
|
"when",
|
||
|
|
"which",
|
||
|
|
"whose",
|
||
|
|
"there",
|
||
|
|
"their",
|
||
|
|
"these",
|
||
|
|
"those",
|
||
|
|
"about",
|
||
|
|
"above",
|
||
|
|
"below",
|
||
|
|
"after",
|
||
|
|
"before",
|
||
|
|
"again",
|
||
|
|
"further",
|
||
|
|
"between",
|
||
|
|
"during",
|
||
|
|
"until",
|
||
|
|
"while",
|
||
|
|
"could",
|
||
|
|
"would",
|
||
|
|
"should",
|
||
|
|
"shall",
|
||
|
|
"might",
|
||
|
|
# Common short stopwords that slip past length >= 4 but carry no
|
||
|
|
# domain meaning, so they shouldn't contribute to na_check hits.
|
||
|
|
"this",
|
||
|
|
"that",
|
||
|
|
"with",
|
||
|
|
"from",
|
||
|
|
"into",
|
||
|
|
"have",
|
||
|
|
"your",
|
||
|
|
"more",
|
||
|
|
"only",
|
||
|
|
"such",
|
||
|
|
"than",
|
||
|
|
"they",
|
||
|
|
"them",
|
||
|
|
"also",
|
||
|
|
"each",
|
||
|
|
"very",
|
||
|
|
"some",
|
||
|
|
"most",
|
||
|
|
"many",
|
||
|
|
"much",
|
||
|
|
"into",
|
||
|
|
"been",
|
||
|
|
"were",
|
||
|
|
"will",
|
||
|
|
"must",
|
||
|
|
# Contract-boilerplate vocabulary. These appear in nearly every
|
||
|
|
# healthcare contract regardless of which field is being scored,
|
||
|
|
# so they're useless as na_check evidence and would otherwise
|
||
|
|
# flag every doc as suspicious.
|
||
|
|
"contract",
|
||
|
|
"contracts",
|
||
|
|
"contractual",
|
||
|
|
"agreement",
|
||
|
|
"agreements",
|
||
|
|
"document",
|
||
|
|
"documents",
|
||
|
|
"provider",
|
||
|
|
"providers",
|
||
|
|
"payer",
|
||
|
|
"payers",
|
||
|
|
"party",
|
||
|
|
"parties",
|
||
|
|
"service",
|
||
|
|
"services",
|
||
|
|
"section",
|
||
|
|
"sections",
|
||
|
|
"article",
|
||
|
|
"articles",
|
||
|
|
"paragraph",
|
||
|
|
"term",
|
||
|
|
"terms",
|
||
|
|
"covered",
|
||
|
|
# Prompt-instruction noise (these talk about the prompt, not the
|
||
|
|
# contract, so they're not useful evidence either way).
|
||
|
|
"value",
|
||
|
|
"field",
|
||
|
|
"given",
|
||
|
|
"named",
|
||
|
|
"answer",
|
||
|
|
"extract",
|
||
|
|
"extracted",
|
||
|
|
"return",
|
||
|
|
"follow",
|
||
|
|
"rules",
|
||
|
|
"guidelines",
|
||
|
|
"instructions",
|
||
|
|
"letters",
|
||
|
|
"numeric",
|
||
|
|
"alphanumeric",
|
||
|
|
"include",
|
||
|
|
"including",
|
||
|
|
"format",
|
||
|
|
"string",
|
||
|
|
"below",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
_KEYWORD_RE = re.compile(r"[A-Za-z][A-Za-z\-]{3,}")
|
||
|
|
|
||
|
|
|
||
|
|
def _field_keywords(field_name: str, field_prompt: str = "") -> list[str]:
|
||
|
|
"""Keyword set for the na_check signal.
|
||
|
|
|
||
|
|
Uses the field name as the keyword source. Field names like
|
||
|
|
CONTRACT_AMENDMENT_NUM are terse and domain-specific by design --
|
||
|
|
split on underscores, drop boilerplate ("contract"), drop short
|
||
|
|
tokens ("num"), and what's left is the actual concept ("amendment").
|
||
|
|
That's a far cleaner anti-hallucination signal than scanning a
|
||
|
|
verbose prompt that drags in generic English ("written", "letters",
|
||
|
|
"title", "numeric") that appears in every contract regardless.
|
||
|
|
|
||
|
|
The prompt is a TRUE fallback: only consulted when the field name
|
||
|
|
yields nothing useful. For every HSC field today, the field name
|
||
|
|
produces at least one keyword, so the prompt never runs in practice.
|
||
|
|
Caps the result at 25 keywords.
|
||
|
|
"""
|
||
|
|
seen: dict[str, None] = {}
|
||
|
|
|
||
|
|
def _add(token: str) -> bool:
|
||
|
|
"""Add a normalized token; return True when the cap is reached."""
|
||
|
|
tok = token.lower()
|
||
|
|
if len(tok) < 4 or tok in _STOPWORDS or tok in seen:
|
||
|
|
return False
|
||
|
|
seen[tok] = None
|
||
|
|
return len(seen) >= 25
|
||
|
|
|
||
|
|
# 1. Field name -- preferred source.
|
||
|
|
if field_name:
|
||
|
|
for tok in re.split(r"[_\W]+", field_name):
|
||
|
|
if _add(tok):
|
||
|
|
return list(seen.keys())
|
||
|
|
|
||
|
|
# 2. Prompt -- only as fallback when the field name had nothing
|
||
|
|
# useful to contribute (rare in practice).
|
||
|
|
if not seen and field_prompt:
|
||
|
|
for tok in _KEYWORD_RE.findall(field_prompt):
|
||
|
|
if _add(tok):
|
||
|
|
return list(seen.keys())
|
||
|
|
|
||
|
|
return list(seen.keys())
|
||
|
|
|
||
|
|
|
||
|
|
def score_field(
|
||
|
|
field_name: str,
|
||
|
|
value: object,
|
||
|
|
metadata: dict | None,
|
||
|
|
contract_text: str = "",
|
||
|
|
field_prompt: str = "",
|
||
|
|
) -> float:
|
||
|
|
"""Compute the rule-based confidence score for a single field.
|
||
|
|
|
||
|
|
Always returns a finite float in [0.0, 1.0]. On any unexpected error,
|
||
|
|
falls back to the LLM-stated confidence (or 0.5) and logs a warning --
|
||
|
|
a bad rule score must never crash the pipeline.
|
||
|
|
"""
|
||
|
|
metadata = metadata or {}
|
||
|
|
try:
|
||
|
|
field_type = ft.get_field_type(field_name)
|
||
|
|
weights = ft.get_weight_preset(field_type)
|
||
|
|
|
||
|
|
retrieved_text = str(metadata.get("retrieved_chunks_text") or "")
|
||
|
|
keywords = _field_keywords(field_name, field_prompt)
|
||
|
|
|
||
|
|
scores: dict[str, float] = {}
|
||
|
|
|
||
|
|
if weights.get(ft.EXACT, 0) > 0:
|
||
|
|
v = sig.signal_exact(value, contract_text or retrieved_text)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.EXACT] = v
|
||
|
|
|
||
|
|
if weights.get(ft.TOKEN_OVERLAP, 0) > 0:
|
||
|
|
v = sig.signal_token_overlap(value, contract_text or retrieved_text)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.TOKEN_OVERLAP] = v
|
||
|
|
|
||
|
|
if weights.get(ft.FUZZY, 0) > 0:
|
||
|
|
# Fuzzy is O(n*m); keep it scoped to retrieved chunks when present.
|
||
|
|
haystack = retrieved_text or contract_text
|
||
|
|
v = sig.signal_fuzzy(value, haystack)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.FUZZY] = v
|
||
|
|
|
||
|
|
if weights.get(ft.NUMBER_MATCH, 0) > 0:
|
||
|
|
v = sig.signal_number_match(value, contract_text or retrieved_text)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.NUMBER_MATCH] = v
|
||
|
|
|
||
|
|
if weights.get(ft.REGEX, 0) > 0:
|
||
|
|
v = sig.signal_regex(value, field_type)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.REGEX] = v
|
||
|
|
|
||
|
|
if weights.get(ft.GROUNDING, 0) > 0:
|
||
|
|
v = sig.signal_grounding(value, retrieved_text)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.GROUNDING] = v
|
||
|
|
|
||
|
|
if weights.get(ft.NA_CHECK, 0) > 0:
|
||
|
|
v = sig.signal_na_check(value, contract_text, keywords)
|
||
|
|
if v is not None:
|
||
|
|
scores[ft.NA_CHECK] = v
|
||
|
|
|
||
|
|
if not scores:
|
||
|
|
# No signal could fire -- fall back to the LLM's self-reported
|
||
|
|
# confidence (which we normalize) or a neutral 0.5.
|
||
|
|
return _coerce_float(metadata.get("confidence")) or 0.5
|
||
|
|
|
||
|
|
# Renormalize weights over the signals that actually fired.
|
||
|
|
total_weight = sum(weights[name] for name in scores)
|
||
|
|
if total_weight <= 0:
|
||
|
|
return _coerce_float(metadata.get("confidence")) or 0.5
|
||
|
|
rule_score = sum(scores[n] * weights[n] for n in scores) / total_weight
|
||
|
|
|
||
|
|
# Light blend with the LLM-stated confidence as a prior. The
|
||
|
|
# rule-based scorer dominates; the prior just smooths very-low
|
||
|
|
# rule scores when the LLM was very confident (and vice versa).
|
||
|
|
llm_conf = _coerce_float(metadata.get("confidence"))
|
||
|
|
if llm_conf is None:
|
||
|
|
final = rule_score
|
||
|
|
else:
|
||
|
|
final = (1 - _LLM_PRIOR_WEIGHT) * rule_score + _LLM_PRIOR_WEIGHT * llm_conf
|
||
|
|
|
||
|
|
# Map raw -> empirically-calibrated confidence using the
|
||
|
|
# per-field-type curve. Identity by default; becomes meaningful
|
||
|
|
# once the curve is backfilled from QC data.
|
||
|
|
final = cal.calibrate(field_type, final)
|
||
|
|
|
||
|
|
return round(max(0.0, min(1.0, final)), 4)
|
||
|
|
|
||
|
|
except Exception as exc: # pragma: no cover - defensive
|
||
|
|
logger.warning(
|
||
|
|
"score_field failed for %s; falling back to LLM-stated conf: %s",
|
||
|
|
field_name,
|
||
|
|
exc,
|
||
|
|
)
|
||
|
|
return _coerce_float((metadata or {}).get("confidence")) or 0.5
|