diff --git a/strix/report/verify.py b/strix/report/verify.py index d8e2f0b91..854d641f1 100644 --- a/strix/report/verify.py +++ b/strix/report/verify.py @@ -13,6 +13,7 @@ from __future__ import annotations import json import logging +import math from typing import TYPE_CHECKING, Any from agents.model_settings import ModelSettings @@ -67,6 +68,15 @@ Respond with ONLY a JSON object: def _verify_extra_args(verify: VerifySettings) -> dict[str, str]: + """Per-call credential + endpoint for a *dedicated* verify model. + + Only applies when STRIX_VERIFY_MODEL is set: the verifier's key/base must + never be overlaid onto the main model when the verifier falls back to it + (that would point verification at the wrong endpoint/account). Mirrors the + dedupe module's gating. + """ + if not verify.model: + return {} extra: dict[str, str] = {} if verify.api_key and verify.api_key.strip(): extra["api_key"] = verify.api_key.strip() @@ -122,10 +132,16 @@ def _parse_verdict(content: str) -> dict[str, Any]: verdict = str(parsed.get("verdict") or "").strip().upper() reason = str(parsed.get("reason") or "")[:500] + # The model output is untrusted (no schema is enforced on the response), so + # a confidence outside [0, 1] — 100, 1e999, -5, nan — must not be able to + # authorize suppression. Anything non-finite or out of range collapses to + # 0.0, which fails the min_confidence gate and emits (fail-open). try: confidence = float(parsed.get("confidence", 0.0)) except (TypeError, ValueError): confidence = 0.0 + if not math.isfinite(confidence) or not (0.0 <= confidence <= 1.0): + confidence = 0.0 return {"verdict": verdict, "confidence": confidence, "reason": reason} diff --git a/tests/test_verify.py b/tests/test_verify.py index 5d7289757..20e40a0a0 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -83,6 +83,31 @@ def test_parse_verdict_bad_confidence_defaults_zero(): assert v["confidence"] == 0.0 +@pytest.mark.parametrize("raw", ["100", "1e999", "-5", "1.0001", "NaN", "-0.01"]) +def test_parse_verdict_out_of_range_confidence_collapses_to_zero(raw: str): + # Untrusted output: any confidence outside [0,1] or non-finite must not be + # able to authorize suppression — it collapses to 0.0 (emits, fail-open). + v = verify._parse_verdict(f'{{"verdict": "FALSE_POSITIVE", "confidence": {raw}}}') + assert v["confidence"] == 0.0 + + +def test_extra_args_gated_on_dedicated_model(monkeypatch: pytest.MonkeyPatch): + # Verifier creds must NOT overlay onto the main model when no dedicated + # STRIX_VERIFY_MODEL is set (would point verification at the wrong endpoint). + monkeypatch.setenv("VERIFY_LLM_API_KEY", "sk-verify") + monkeypatch.setenv("VERIFY_LLM_API_BASE", "https://verify.example") + _reset_cache() + verify_settings = loader.load_settings().verify + assert verify_settings.model is None + assert verify._verify_extra_args(verify_settings) == {} + + # With a dedicated model, the creds ARE passed through. + monkeypatch.setenv("STRIX_VERIFY_MODEL", "openai/verifier") + _reset_cache() + verify_settings = loader.load_settings().verify + assert verify._verify_extra_args(verify_settings)["api_key"] == "sk-verify" + + def test_parse_verdict_no_json_raises(): with pytest.raises(ValueError): verify._parse_verdict("I think this is real, sorry no JSON") @@ -186,3 +211,15 @@ async def test_unparseable_response_emits(monkeypatch: pytest.MonkeyPatch): out = await _run_with_stubbed_model(monkeypatch, "no json here", "critical") assert out["reject"] is False assert out["verdict"] == "ERROR" + + +async def test_out_of_range_confidence_does_not_suppress(monkeypatch: pytest.MonkeyPatch): + # A FALSE_POSITIVE with confidence=100 must NOT authorize suppression — the + # clamp collapses it to 0.0, which fails the threshold and emits. + out = await _run_with_stubbed_model( + monkeypatch, + '{"verdict": "FALSE_POSITIVE", "confidence": 100, "reason": "bogus"}', + "critical", + ) + assert out["reject"] is False + assert out["confidence"] == 0.0