fix(eval): require finite gateway startup budgets

This commit is contained in:
Claude 2026-09-05 11:03:23 +00:00
parent fc61507da5
commit de3131fed8
3 changed files with 18 additions and 4 deletions

View file

@ -358,12 +358,23 @@ def test_gateway_startup_budget_outlives_a_cold_litellm_import(monkeypatch, tmp_
monkeypatch.setenv(GATEWAY_READY_TIMEOUT_ENV, "42.5")
assert gateway_ready_timeout_s() == 42.5
for bad in ("0", "-1", "soon"):
for bad in ("0", "-1", "soon", "nan", "inf", "-inf", "1e999"):
monkeypatch.setenv(GATEWAY_READY_TIMEOUT_ENV, bad)
with pytest.raises(ValueError, match=GATEWAY_READY_TIMEOUT_ENV):
gateway_ready_timeout_s()
@pytest.mark.parametrize("timeout", [0.0, -1.0, float("nan"), float("inf"), float("-inf")])
def test_gateway_rejects_invalid_explicit_readiness_budgets(tmp_path: Path, timeout: float) -> None:
with pytest.raises(ValueError, match="finite and positive"):
OpenAIGateway(
openai_api_key="sk-offline-test",
model_names=["gpt-4.1"],
work_dir=tmp_path / "gw",
ready_timeout_s=timeout,
)
def test_gateway_readiness_timeout_reports_the_proxy_log_and_the_override(tmp_path: Path) -> None:
gateway = OpenAIGateway(
openai_api_key="sk-openai-secret",

View file

@ -11,6 +11,7 @@ proxy process — never in the sandboxed agent environment.
from __future__ import annotations
import argparse
import math
import os
import re
import secrets
@ -60,8 +61,8 @@ def gateway_ready_timeout_s() -> float:
value = float(raw)
except ValueError as exc:
raise ValueError(f"{GATEWAY_READY_TIMEOUT_ENV} must be a number of seconds, not {raw!r}") from exc
if value <= 0:
raise ValueError(f"{GATEWAY_READY_TIMEOUT_ENV} must be positive, not {raw!r}")
if not math.isfinite(value) or value <= 0:
raise ValueError(f"{GATEWAY_READY_TIMEOUT_ENV} must be finite and positive, not {raw!r}")
return value
@ -268,6 +269,8 @@ class OpenAIGateway(AbstractContextManager["OpenAIGateway"]):
self.model_names = tuple(model_names)
self.work_dir = work_dir
self.ready_timeout_s = gateway_ready_timeout_s() if ready_timeout_s is None else ready_timeout_s
if not math.isfinite(self.ready_timeout_s) or self.ready_timeout_s <= 0:
raise ValueError("gateway readiness timeout must be finite and positive")
self.auth_token = secrets.token_hex(16)
self.port = _free_loopback_port()
self.base_url = f"http://127.0.0.1:{self.port}"

View file

@ -5,7 +5,7 @@ Usage:
--tasks workflow_bench/tasks.scenarios.yaml --runs 3 \
--model claude-sonnet-4-20250514
Each task runs in a fresh detached git worktree of the target repo, once per
Each task runs in a fresh self-contained clone of the target repo, once per
arm per run:
* ``workflow`` two headless Claude Code sessions: gitnexus-plan, then