fix(verify): gate verify creds on a dedicated model + clamp confidence

Two issues from review:

- `_verify_extra_args` overlaid VERIFY_LLM_API_KEY/_API_BASE even when no
  dedicated STRIX_VERIFY_MODEL was set, pointing the fallback main-model request
  at the wrong endpoint/account. Gate on `verify.model`, mirroring dedupe.

- `_parse_verdict` accepted an out-of-range/non-finite confidence (100, 1e999,
  nan) from the untrusted model output, which could clear the suppression
  threshold and drop a finding. Clamp to [0, 1]; anything else collapses to 0.0
  (fails the gate, emits) — preserving the fail-open contract.

Adds tests for both (including confidence=100 not suppressing end-to-end).
This commit is contained in:
Sean Turner 2026-08-06 23:30:56 +01:00
parent 312c774802
commit 596da7dd7a
No known key found for this signature in database
2 changed files with 53 additions and 0 deletions

View file

@ -13,6 +13,7 @@ from __future__ import annotations
import json
import logging
import math
from typing import TYPE_CHECKING, Any
from agents.model_settings import ModelSettings
@ -67,6 +68,15 @@ Respond with ONLY a JSON object:
def _verify_extra_args(verify: VerifySettings) -> dict[str, str]:
"""Per-call credential + endpoint for a *dedicated* verify model.
Only applies when STRIX_VERIFY_MODEL is set: the verifier's key/base must
never be overlaid onto the main model when the verifier falls back to it
(that would point verification at the wrong endpoint/account). Mirrors the
dedupe module's gating.
"""
if not verify.model:
return {}
extra: dict[str, str] = {}
if verify.api_key and verify.api_key.strip():
extra["api_key"] = verify.api_key.strip()
@ -122,10 +132,16 @@ def _parse_verdict(content: str) -> dict[str, Any]:
verdict = str(parsed.get("verdict") or "").strip().upper()
reason = str(parsed.get("reason") or "")[:500]
# The model output is untrusted (no schema is enforced on the response), so
# a confidence outside [0, 1] — 100, 1e999, -5, nan — must not be able to
# authorize suppression. Anything non-finite or out of range collapses to
# 0.0, which fails the min_confidence gate and emits (fail-open).
try:
confidence = float(parsed.get("confidence", 0.0))
except (TypeError, ValueError):
confidence = 0.0
if not math.isfinite(confidence) or not (0.0 <= confidence <= 1.0):
confidence = 0.0
return {"verdict": verdict, "confidence": confidence, "reason": reason}

View file

@ -83,6 +83,31 @@ def test_parse_verdict_bad_confidence_defaults_zero():
assert v["confidence"] == 0.0
@pytest.mark.parametrize("raw", ["100", "1e999", "-5", "1.0001", "NaN", "-0.01"])
def test_parse_verdict_out_of_range_confidence_collapses_to_zero(raw: str):
# Untrusted output: any confidence outside [0,1] or non-finite must not be
# able to authorize suppression — it collapses to 0.0 (emits, fail-open).
v = verify._parse_verdict(f'{{"verdict": "FALSE_POSITIVE", "confidence": {raw}}}')
assert v["confidence"] == 0.0
def test_extra_args_gated_on_dedicated_model(monkeypatch: pytest.MonkeyPatch):
# Verifier creds must NOT overlay onto the main model when no dedicated
# STRIX_VERIFY_MODEL is set (would point verification at the wrong endpoint).
monkeypatch.setenv("VERIFY_LLM_API_KEY", "sk-verify")
monkeypatch.setenv("VERIFY_LLM_API_BASE", "https://verify.example")
_reset_cache()
verify_settings = loader.load_settings().verify
assert verify_settings.model is None
assert verify._verify_extra_args(verify_settings) == {}
# With a dedicated model, the creds ARE passed through.
monkeypatch.setenv("STRIX_VERIFY_MODEL", "openai/verifier")
_reset_cache()
verify_settings = loader.load_settings().verify
assert verify._verify_extra_args(verify_settings)["api_key"] == "sk-verify"
def test_parse_verdict_no_json_raises():
with pytest.raises(ValueError):
verify._parse_verdict("I think this is real, sorry no JSON")
@ -186,3 +211,15 @@ async def test_unparseable_response_emits(monkeypatch: pytest.MonkeyPatch):
out = await _run_with_stubbed_model(monkeypatch, "no json here", "critical")
assert out["reject"] is False
assert out["verdict"] == "ERROR"
async def test_out_of_range_confidence_does_not_suppress(monkeypatch: pytest.MonkeyPatch):
# A FALSE_POSITIVE with confidence=100 must NOT authorize suppression — the
# clamp collapses it to 0.0, which fails the threshold and emits.
out = await _run_with_stubbed_model(
monkeypatch,
'{"verdict": "FALSE_POSITIVE", "confidence": 100, "reason": "bogus"}',
"critical",
)
assert out["reject"] is False
assert out["confidence"] == 0.0