diff --git a/eval/tests/test_model_gateway.py b/eval/tests/test_model_gateway.py index 3553b1bcf..51840b6e9 100644 --- a/eval/tests/test_model_gateway.py +++ b/eval/tests/test_model_gateway.py @@ -358,12 +358,23 @@ def test_gateway_startup_budget_outlives_a_cold_litellm_import(monkeypatch, tmp_ monkeypatch.setenv(GATEWAY_READY_TIMEOUT_ENV, "42.5") assert gateway_ready_timeout_s() == 42.5 - for bad in ("0", "-1", "soon"): + for bad in ("0", "-1", "soon", "nan", "inf", "-inf", "1e999"): monkeypatch.setenv(GATEWAY_READY_TIMEOUT_ENV, bad) with pytest.raises(ValueError, match=GATEWAY_READY_TIMEOUT_ENV): gateway_ready_timeout_s() +@pytest.mark.parametrize("timeout", [0.0, -1.0, float("nan"), float("inf"), float("-inf")]) +def test_gateway_rejects_invalid_explicit_readiness_budgets(tmp_path: Path, timeout: float) -> None: + with pytest.raises(ValueError, match="finite and positive"): + OpenAIGateway( + openai_api_key="sk-offline-test", + model_names=["gpt-4.1"], + work_dir=tmp_path / "gw", + ready_timeout_s=timeout, + ) + + def test_gateway_readiness_timeout_reports_the_proxy_log_and_the_override(tmp_path: Path) -> None: gateway = OpenAIGateway( openai_api_key="sk-openai-secret", diff --git a/eval/workflow_bench/model_gateway.py b/eval/workflow_bench/model_gateway.py index 203aed377..e44da55ec 100644 --- a/eval/workflow_bench/model_gateway.py +++ b/eval/workflow_bench/model_gateway.py @@ -11,6 +11,7 @@ proxy process — never in the sandboxed agent environment. from __future__ import annotations import argparse +import math import os import re import secrets @@ -60,8 +61,8 @@ def gateway_ready_timeout_s() -> float: value = float(raw) except ValueError as exc: raise ValueError(f"{GATEWAY_READY_TIMEOUT_ENV} must be a number of seconds, not {raw!r}") from exc - if value <= 0: - raise ValueError(f"{GATEWAY_READY_TIMEOUT_ENV} must be positive, not {raw!r}") + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{GATEWAY_READY_TIMEOUT_ENV} must be finite and positive, not {raw!r}") return value @@ -268,6 +269,8 @@ class OpenAIGateway(AbstractContextManager["OpenAIGateway"]): self.model_names = tuple(model_names) self.work_dir = work_dir self.ready_timeout_s = gateway_ready_timeout_s() if ready_timeout_s is None else ready_timeout_s + if not math.isfinite(self.ready_timeout_s) or self.ready_timeout_s <= 0: + raise ValueError("gateway readiness timeout must be finite and positive") self.auth_token = secrets.token_hex(16) self.port = _free_loopback_port() self.base_url = f"http://127.0.0.1:{self.port}" diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index 86031bb82..82185186c 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -5,7 +5,7 @@ Usage: --tasks workflow_bench/tasks.scenarios.yaml --runs 3 \ --model claude-sonnet-4-20250514 -Each task runs in a fresh detached git worktree of the target repo, once per +Each task runs in a fresh self-contained clone of the target repo, once per arm per run: * ``workflow`` — two headless Claude Code sessions: gitnexus-plan, then