fix(eval): start OpenAI gateway via litellm console script

python -m litellm fails on 1.87 (no __main__); use the venv console entry and fall back through VIRTUAL_ENV under uv run.

Co-authored-by: Cursor <cursoragent@cursor.com>

eval/tests/test_model_gateway.py

@@ -12,6 +12,7 @@ from workflow_bench.model_gateway import (
     claude_gateway_model_env,
     credential_secrets,
     is_openai_model,
+    litellm_proxy_argv,
     openai_backend_model,
     openai_litellm_config,
     resolve_model_access,
@@ -134,6 +135,35 @@ def test_openai_backend_model_preserves_openai_prefix() -> None:
     assert openai_backend_model("openai/gpt-4.1") == "openai/gpt-4.1"
 
 
+def test_litellm_proxy_argv_uses_console_script_not_python_module(tmp_path: Path, monkeypatch) -> None:
+    # litellm 1.87 ships a console script and no litellm.__main__, so
+    # `python -m litellm` dies before the health check. Pin the supported argv.
+    # Under `uv run`, sys.executable is the base CPython — the script lives in
+    # VIRTUAL_ENV/bin instead.
+    python = tmp_path / "base" / "python"
+    venv_bin = tmp_path / "venv" / "bin"
+    python.parent.mkdir(parents=True)
+    venv_bin.mkdir(parents=True)
+    litellm = venv_bin / "litellm"
+    python.write_text("#!/bin/sh\n")
+    litellm.write_text("#!/bin/sh\n")
+    python.chmod(0o755)
+    litellm.chmod(0o755)
+    monkeypatch.setenv("VIRTUAL_ENV", str(tmp_path / "venv"))
+    monkeypatch.delenv("PATH", raising=False)
+    config = tmp_path / "litellm.yaml"
+    config.write_text("model_list: []\n")
+    argv = litellm_proxy_argv(
+        config=config,
+        host="127.0.0.1",
+        port=4010,
+        python_executable=str(python),
+    )
+    assert argv[0] == str(litellm.resolve())
+    assert "-m" not in argv
+    assert argv[1:] == ["--config", str(config), "--host", "127.0.0.1", "--port", "4010"]
+
+
 def test_anthropic_api_key_prefers_the_named_env_and_keeps_the_legacy_alias(monkeypatch) -> None:
     monkeypatch.delenv("GITNEXUS_BENCH_ANTHROPIC_API_KEY", raising=False)
     monkeypatch.setenv("GITNEXUS_BENCH_AUTH_TOKEN", "legacy-secret")

eval/workflow_bench/model_gateway.py

@@ -14,6 +14,7 @@ import argparse
 import os
 import re
 import secrets
+import shutil
 import socket
 import subprocess
 import sys
@@ -182,6 +183,49 @@ def _free_loopback_port() -> int:
         return int(sock.getsockname()[1])
 
 
+def litellm_proxy_argv(
+    *,
+    config: Path,
+    host: str,
+    port: int,
+    python_executable: str | None = None,
+) -> list[str]:
+    """Build the LiteLLM proxy argv for this interpreter.
+
+    ``python -m litellm`` fails on current releases (no ``litellm.__main__``).
+    Prefer the console script next to ``sys.executable``; under ``uv run`` that
+    path is the base CPython, so also honor ``VIRTUAL_ENV`` and ``PATH``.
+    """
+
+    python = Path(python_executable or sys.executable).resolve()
+    candidates: list[Path] = [python.with_name("litellm")]
+    virtual_env = (os.environ.get("VIRTUAL_ENV") or "").strip()
+    if virtual_env:
+        candidates.append(Path(virtual_env) / "bin" / "litellm")
+    which = shutil.which("litellm")
+    if which:
+        candidates.append(Path(which))
+    litellm_bin: Path | None = None
+    for candidate in candidates:
+        if candidate.is_file() and os.access(candidate, os.X_OK):
+            litellm_bin = candidate.resolve()
+            break
+    if litellm_bin is None:
+        raise RuntimeError(
+            f"LiteLLM console script missing next to {python} "
+            "(install litellm[proxy]; do not use python -m litellm)"
+        )
+    return [
+        str(litellm_bin),
+        "--config",
+        str(config),
+        "--host",
+        host,
+        "--port",
+        str(port),
+    ]
+
+
 class OpenAIGateway(AbstractContextManager["OpenAIGateway"]):
     def __init__(
         self,
@@ -214,17 +258,11 @@ class OpenAIGateway(AbstractContextManager["OpenAIGateway"]):
         }
         try:
             self._process = subprocess.Popen(
-                [
-                    sys.executable,
-                    "-m",
-                    "litellm",
-                    "--config",
-                    str(config),
-                    "--host",
-                    "127.0.0.1",
-                    "--port",
-                    str(self.port),
-                ],
+                litellm_proxy_argv(
+                    config=config,
+                    host="127.0.0.1",
+                    port=self.port,
+                ),
                 cwd=str(self.work_dir),
                 env=env,
                 stdin=subprocess.DEVNULL,