diff --git a/eval/tests/test_model_gateway.py b/eval/tests/test_model_gateway.py index 0a362b0a6..efb1b19df 100644 --- a/eval/tests/test_model_gateway.py +++ b/eval/tests/test_model_gateway.py @@ -12,6 +12,7 @@ from workflow_bench.model_gateway import ( claude_gateway_model_env, credential_secrets, is_openai_model, + litellm_proxy_argv, openai_backend_model, openai_litellm_config, resolve_model_access, @@ -134,6 +135,35 @@ def test_openai_backend_model_preserves_openai_prefix() -> None: assert openai_backend_model("openai/gpt-4.1") == "openai/gpt-4.1" +def test_litellm_proxy_argv_uses_console_script_not_python_module(tmp_path: Path, monkeypatch) -> None: + # litellm 1.87 ships a console script and no litellm.__main__, so + # `python -m litellm` dies before the health check. Pin the supported argv. + # Under `uv run`, sys.executable is the base CPython — the script lives in + # VIRTUAL_ENV/bin instead. + python = tmp_path / "base" / "python" + venv_bin = tmp_path / "venv" / "bin" + python.parent.mkdir(parents=True) + venv_bin.mkdir(parents=True) + litellm = venv_bin / "litellm" + python.write_text("#!/bin/sh\n") + litellm.write_text("#!/bin/sh\n") + python.chmod(0o755) + litellm.chmod(0o755) + monkeypatch.setenv("VIRTUAL_ENV", str(tmp_path / "venv")) + monkeypatch.delenv("PATH", raising=False) + config = tmp_path / "litellm.yaml" + config.write_text("model_list: []\n") + argv = litellm_proxy_argv( + config=config, + host="127.0.0.1", + port=4010, + python_executable=str(python), + ) + assert argv[0] == str(litellm.resolve()) + assert "-m" not in argv + assert argv[1:] == ["--config", str(config), "--host", "127.0.0.1", "--port", "4010"] + + def test_anthropic_api_key_prefers_the_named_env_and_keeps_the_legacy_alias(monkeypatch) -> None: monkeypatch.delenv("GITNEXUS_BENCH_ANTHROPIC_API_KEY", raising=False) monkeypatch.setenv("GITNEXUS_BENCH_AUTH_TOKEN", "legacy-secret") diff --git a/eval/workflow_bench/model_gateway.py b/eval/workflow_bench/model_gateway.py index 7ea0a4e4e..bdd6a3002 100644 --- a/eval/workflow_bench/model_gateway.py +++ b/eval/workflow_bench/model_gateway.py @@ -14,6 +14,7 @@ import argparse import os import re import secrets +import shutil import socket import subprocess import sys @@ -182,6 +183,49 @@ def _free_loopback_port() -> int: return int(sock.getsockname()[1]) +def litellm_proxy_argv( + *, + config: Path, + host: str, + port: int, + python_executable: str | None = None, +) -> list[str]: + """Build the LiteLLM proxy argv for this interpreter. + + ``python -m litellm`` fails on current releases (no ``litellm.__main__``). + Prefer the console script next to ``sys.executable``; under ``uv run`` that + path is the base CPython, so also honor ``VIRTUAL_ENV`` and ``PATH``. + """ + + python = Path(python_executable or sys.executable).resolve() + candidates: list[Path] = [python.with_name("litellm")] + virtual_env = (os.environ.get("VIRTUAL_ENV") or "").strip() + if virtual_env: + candidates.append(Path(virtual_env) / "bin" / "litellm") + which = shutil.which("litellm") + if which: + candidates.append(Path(which)) + litellm_bin: Path | None = None + for candidate in candidates: + if candidate.is_file() and os.access(candidate, os.X_OK): + litellm_bin = candidate.resolve() + break + if litellm_bin is None: + raise RuntimeError( + f"LiteLLM console script missing next to {python} " + "(install litellm[proxy]; do not use python -m litellm)" + ) + return [ + str(litellm_bin), + "--config", + str(config), + "--host", + host, + "--port", + str(port), + ] + + class OpenAIGateway(AbstractContextManager["OpenAIGateway"]): def __init__( self, @@ -214,17 +258,11 @@ class OpenAIGateway(AbstractContextManager["OpenAIGateway"]): } try: self._process = subprocess.Popen( - [ - sys.executable, - "-m", - "litellm", - "--config", - str(config), - "--host", - "127.0.0.1", - "--port", - str(self.port), - ], + litellm_proxy_argv( + config=config, + host="127.0.0.1", + port=self.port, + ), cwd=str(self.work_dir), env=env, stdin=subprocess.DEVNULL,