diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 47355f328dd..ed1447e4e65 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -489,7 +489,7 @@ lite codex exec "summarize the repo" Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process. -The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, and older versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). +The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, and older versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Options (these belong to the wrapper, so put them before the agent's own flags): diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index baa21996c7e..ce416ef237b 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -3,10 +3,13 @@ import shutil import subprocess import sys from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import Final import click import requests +from pydantic import BaseModel, TypeAdapter, ValidationError from .auth import context_secret_vault, get_stored_api_key, login from .cmd_quoting import quote_for_cmd @@ -20,6 +23,12 @@ ENABLE_GATEWAY_MODEL_DISCOVERY_ENV: Final = "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DI ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE: Final = "1" OPENAI_BASE_URL_ENV: Final = "OPENAI_BASE_URL" OPENAI_API_KEY_ENV: Final = "OPENAI_API_KEY" +OPENCODE_CONFIG_CONTENT_ENV: Final = "OPENCODE_CONFIG_CONTENT" +OPENCODE_PROVIDER_ID: Final = "litellm" +OPENCODE_PROVIDER_NAME: Final = "LiteLLM" +OPENCODE_PROVIDER_NPM: Final = "@ai-sdk/openai-compatible" + +_SKIP_VERIFY_FLAG: Final = "--skip-verify" PROFILE_ANTHROPIC: Final = "anthropic" PROFILE_OPENAI: Final = "openai" @@ -131,6 +140,139 @@ def agent_launch_args(command: str, base_url: str) -> list[str]: return builder(base_url) if builder else [] +class ListedModel(BaseModel): + """The fields of a /v1/models entry that an OpenCode model entry is built from.""" + + id: str + mode: str | None = None + max_input_tokens: int | None = None + max_output_tokens: int | None = None + + +class _ModelListing(BaseModel): + data: tuple[ListedModel, ...] + + +_MODEL_LISTING: Final = TypeAdapter(_ModelListing) +_OPENCODE_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"}) +_NO_EXTRA_ENV: Final[Mapping[str, str]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class ModelSyncSkipped: + reason: str + + +class _OpenCodeLimit(BaseModel): + context: int + output: int + + +class _OpenCodeModel(BaseModel): + name: str + limit: _OpenCodeLimit | None = None + + +class _OpenCodeProviderOptions(BaseModel): + baseURL: str + apiKey: str + + +class _OpenCodeProvider(BaseModel): + npm: str + name: str + options: _OpenCodeProviderOptions + models: Mapping[str, _OpenCodeModel] + + +class _OpenCodeConfig(BaseModel): + provider: Mapping[str, _OpenCodeProvider] + + +def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel: + if model.max_input_tokens is None or model.max_output_tokens is None: + return _OpenCodeModel(name=model.id) + return _OpenCodeModel( + name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + ) + + +def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str: + """OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`. + + One model entry per chat-capable /v1/models row (mode chat, responses, or + unknown), so OpenCode's model picker mirrors what the key can call. The key + is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so + it never lands in the config text. OpenCode merges this inline config over + the user's own files, leaving unrelated keys and providers untouched. + """ + chat_models: Final = tuple(m for m in models if m.mode is None or m.mode in _OPENCODE_CHAT_MODES) + provider: Final = _OpenCodeProvider( + npm=OPENCODE_PROVIDER_NPM, + name=OPENCODE_PROVIDER_NAME, + options=_OpenCodeProviderOptions( + baseURL=base_url.rstrip("/") + "/v1", + apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}", + ), + models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}), + ) + config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider})) + return config.model_dump_json(exclude_none=True) + + +def opencode_model_sync_env( + base_env: Mapping[str, str], + base_url: str, + api_key: str, + *, + get: Callable[..., requests.Response] = requests.get, +) -> Mapping[str, str] | ModelSyncSkipped: + """Env addition that hands OpenCode the proxy's model list, or why it was skipped. + + Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT. + An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a + failed fetch is reported rather than raised: OpenCode still launches on the + plain OPENAI_* env, just without a synced model list. + """ + if OPENCODE_CONFIG_CONTENT_ENV in base_env: + return ModelSyncSkipped(f"{OPENCODE_CONFIG_CONTENT_ENV} is already set") + url: Final = base_url.rstrip("/") + "/v1/models" + try: + resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10) + except requests.RequestException as e: + return ModelSyncSkipped(f"could not reach {url}: {e}") + if resp.status_code != 200: + return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}") + try: + listing: Final = _MODEL_LISTING.validate_json(resp.content) + except ValidationError: + return ModelSyncSkipped(f"{url} returned an unexpected body") + return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing.data)}) + + +def agent_model_sync_env( + command: str, + base_env: Mapping[str, str], + base_url: str, + api_key: str, + skip_verify: bool, + *, + get: Callable[..., requests.Response] = requests.get, +) -> Mapping[str, str] | ModelSyncSkipped: + """Extra env an agent needs to see the proxy's model list. + + Only OpenCode needs one: Claude Code discovers models through + CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY and Codex takes the model by name. + skip_verify means the caller wants no pre-launch proxy call at all, so the + listing is skipped too rather than hanging on an offline proxy. + """ + if os.path.basename(command) != "opencode": + return _NO_EXTRA_ENV + if skip_verify: + return ModelSyncSkipped(f"{_SKIP_VERIFY_FLAG} was passed") + return opencode_model_sync_env(base_env, base_url, api_key, get=get) + + def verify_proxy_key( base_url: str, api_key: str, @@ -246,6 +388,10 @@ def _restore_controlling_terminal() -> None: os.close(fd) +def _warn(message: str) -> None: + click.echo(message, err=True) + + def run_agent( base_url: str, api_key: str, @@ -255,6 +401,10 @@ def run_agent( base_env: Mapping[str, str] | None = None, which: Callable[[str], str | None] = shutil.which, verify: Callable[[str, str], None] = verify_proxy_key, + sync_models: Callable[[str, Mapping[str, str], str, str, bool], Mapping[str, str] | ModelSyncSkipped] = ( + agent_model_sync_env + ), + warn: Callable[[str], None] = _warn, launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off, reattach_terminal: Callable[[], None] | None = None, ) -> None: @@ -262,13 +412,15 @@ def run_agent( On success this never returns: POSIX replaces the current process, Windows waits on the agent and exits with its status. Raises AgentRunError for - missing binaries, an unreachable proxy, or a rejected key. + missing binaries, an unreachable proxy, or a rejected key. The model list is + synced only once the key check passed, so an unreachable proxy costs one + timeout rather than two, and --skip-verify keeps the launch fully offline. reattach_terminal, when given, runs just before handoff to restore stdin. """ if not command: raise AgentRunError("Nothing to run.") - _, profiles = agent_profile(command[0]) + display_name, profiles = agent_profile(command[0]) binary: Final = which(command[0]) if binary is None: docs: Final = _INSTALL_DOCS.get(os.path.basename(command[0])) @@ -278,11 +430,16 @@ def run_agent( if not skip_verify: verify(base_url, api_key) - env: Final = build_agent_env( - base_env if base_env is not None else os.environ, - base_url, - api_key, - profiles, + env_before_sync: Final = base_env if base_env is not None else os.environ + synced: Final = sync_models(command[0], env_before_sync, base_url, api_key, skip_verify) + if isinstance(synced, ModelSyncSkipped): + warn(f"litellm: not syncing {display_name} models from the proxy: {synced.reason}") + + env: Final = MappingProxyType( + { + **build_agent_env(env_before_sync, base_url, api_key, profiles), + **(_NO_EXTRA_ENV if isinstance(synced, ModelSyncSkipped) else synced), + } ) extra_args: Final = agent_launch_args(command[0], base_url) if reattach_terminal is not None: @@ -365,10 +522,15 @@ def agent_commands() -> tuple[click.Command, ...]: __all__ = [ "AgentRunError", + "ListedModel", + "ModelSyncSkipped", "agent_commands", "agent_launch_args", + "agent_model_sync_env", "agent_profile", "build_agent_env", + "opencode_model_sync_env", + "opencode_provider_config", "resolve_api_key", "run_agent", "verify_proxy_key", diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index 62b94e948be..5b99d368cbb 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -1,4 +1,5 @@ import inspect +import json import os import sys from unittest.mock import patch @@ -12,13 +13,16 @@ from click.testing import CliRunner from litellm.proxy.client.cli.commands.agents import ( AgentRunError, + ModelSyncSkipped, _hand_off, _replace_process, _spawn_and_wait, agent_commands, agent_launch_args, + agent_model_sync_env, agent_profile, build_agent_env, + opencode_model_sync_env, run_agent, verify_proxy_key, ) @@ -35,8 +39,9 @@ def _default_of(func, param): class _FakeResponse: - def __init__(self, status_code): + def __init__(self, status_code, body=None): self.status_code = status_code + self.content = json.dumps(body).encode() if body is not None else b"" class _Recorder: @@ -200,7 +205,259 @@ class TestVerifyProxyKey: ) +class TestOpencodeModelSync: + @staticmethod + def _listing(*models): + return {"object": "list", "data": list(models)} + + def _sync(self, listing, base_env=None, base_url="http://localhost:4000/"): + captured = {} + + def fake_get(url, headers, timeout): + captured["url"] = url + captured["headers"] = headers + return _FakeResponse(200, listing) + + env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=fake_get) + return captured, env + + def test_declares_proxy_as_litellm_provider_with_listed_models(self): + listing = self._listing( + {"id": "gpt-5.5", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, + {"id": "claude-opus-4-7", "object": "model", "created": 1, "owned_by": "openai"}, + ) + captured, env = self._sync(listing) + + assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["headers"] == {"Authorization": "Bearer sk-key"} + config = json.loads(env["OPENCODE_CONFIG_CONTENT"]) + provider = config["provider"]["litellm"] + assert provider["npm"] == "@ai-sdk/openai-compatible" + assert provider["name"] == "LiteLLM" + assert provider["options"] == { + "baseURL": "http://localhost:4000/v1", + "apiKey": "{env:OPENAI_API_KEY}", + } + assert provider["models"] == { + "gpt-5.5": {"name": "gpt-5.5"}, + "claude-opus-4-7": {"name": "claude-opus-4-7"}, + } + assert "sk-key" not in env["OPENCODE_CONFIG_CONTENT"] + + def test_token_limits_become_opencode_limits(self): + listing = self._listing( + { + "id": "gpt-5.5", + "object": "model", + "created": 1, + "owned_by": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + }, + {"id": "half", "object": "model", "created": 1, "owned_by": "openai", "max_input_tokens": 8192}, + ) + _, env = self._sync(listing) + models = json.loads(env["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] + assert models["gpt-5.5"]["limit"] == {"context": 400000, "output": 128000} + assert "limit" not in models["half"] + + def test_non_chat_models_are_left_out(self): + listing = self._listing( + {"id": "chat", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, + {"id": "resp", "object": "model", "created": 1, "owned_by": "openai", "mode": "responses"}, + {"id": "embed", "object": "model", "created": 1, "owned_by": "openai", "mode": "embedding"}, + {"id": "img", "object": "model", "created": 1, "owned_by": "openai", "mode": "image_generation"}, + ) + _, env = self._sync(listing) + models = json.loads(env["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] + assert set(models) == {"chat", "resp"} + + def test_existing_config_content_is_left_alone(self): + calls = [] + + def fake_get(*a, **k): + calls.append(a) + return _FakeResponse(200, self._listing()) + + result = opencode_model_sync_env( + {"OPENCODE_CONFIG_CONTENT": "{}"}, "http://localhost:4000", "sk-key", get=fake_get + ) + assert isinstance(result, ModelSyncSkipped) + assert "OPENCODE_CONFIG_CONTENT" in result.reason + assert calls == [] + + def test_unreachable_proxy_is_reported_not_raised(self): + def boom(*a, **k): + raise requests.ConnectionError("refused") + + result = opencode_model_sync_env({}, "http://localhost:4000", "sk-key", get=boom) + assert isinstance(result, ModelSyncSkipped) + assert "refused" in result.reason + + def test_non_200_is_reported(self): + result = opencode_model_sync_env( + {}, "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500) + ) + assert isinstance(result, ModelSyncSkipped) + assert "HTTP 500" in result.reason + + def test_unexpected_body_is_reported(self): + result = opencode_model_sync_env( + {}, "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(200, {"data": "nope"}) + ) + assert isinstance(result, ModelSyncSkipped) + assert "unexpected body" in result.reason + + @pytest.mark.parametrize("command", ["claude", "codex", "/usr/bin/claude"]) + def test_only_opencode_syncs(self, command): + def boom(*a, **k): + raise AssertionError("no agent other than opencode should call the proxy") + + assert agent_model_sync_env(command, {}, "http://localhost:4000", "sk-key", False, get=boom) == {} + + def test_skip_verify_keeps_the_launch_offline(self): + def boom(*a, **k): + raise AssertionError("--skip-verify must not touch the proxy") + + result = agent_model_sync_env("opencode", {}, "http://localhost:4000", "sk-key", True, get=boom) + assert isinstance(result, ModelSyncSkipped) + assert "--skip-verify" in result.reason + + def test_full_path_opencode_syncs(self): + listing = self._listing({"id": "m", "object": "model", "created": 1, "owned_by": "x"}) + env = agent_model_sync_env( + "/opt/bin/opencode", + {}, + "http://localhost:4000", + "sk-key", + False, + get=lambda *a, **k: _FakeResponse(200, listing), + ) + assert "m" in json.loads(env["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] + + def test_default_http_client_is_requests_get(self): + assert _default_of(agent_model_sync_env, "get") is requests.get + assert _default_of(opencode_model_sync_env, "get") is requests.get + + class TestRunAgent: + def test_synced_model_config_reaches_the_agent_alongside_profile_env(self): + calls = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["opencode"], + base_env={"HOME": "/home/me"}, + sync_models=lambda *a: {"OPENCODE_CONFIG_CONTENT": '{"provider":{}}'}, + which=lambda name: "/usr/local/bin/opencode", + verify=lambda *a: None, + launcher=lambda p, a, e: calls.update(env=dict(e)), + ) + assert calls["env"]["OPENCODE_CONFIG_CONTENT"] == '{"provider":{}}' + assert calls["env"]["OPENAI_BASE_URL"] == "http://localhost:4000/v1" + assert calls["env"]["OPENAI_API_KEY"] == "sk-key" + assert calls["env"]["HOME"] == "/home/me" + + def test_sync_gets_the_launch_inputs_and_runs_after_verify(self): + order = [] + calls = {} + + def fake_sync(command, base_env, base_url, api_key, skip_verify): + order.append("sync") + calls["args"] = (command, dict(base_env), base_url, api_key, skip_verify) + return {"OPENCODE_CONFIG_CONTENT": '{"provider":{"litellm":{}}}'} + + run_agent( + "http://localhost:4000", + "sk-key", + ["opencode"], + base_env={"HOME": "/home/me"}, + sync_models=fake_sync, + which=lambda name: "/usr/local/bin/opencode", + verify=lambda *a: order.append("verify"), + launcher=lambda p, a, e: order.append("launch"), + ) + assert order == ["verify", "sync", "launch"] + assert calls["args"] == ("opencode", {"HOME": "/home/me"}, "http://localhost:4000", "sk-key", False) + + def test_unreachable_proxy_is_not_asked_for_models(self): + def failing_verify(*a): + raise AgentRunError("Could not reach the LiteLLM proxy") + + def boom(*a): + raise AssertionError("a failed key check must not be followed by a model fetch") + + with pytest.raises(AgentRunError): + run_agent( + "http://localhost:4000", + "sk-key", + ["opencode"], + base_env={}, + sync_models=boom, + which=lambda name: "/usr/local/bin/opencode", + verify=failing_verify, + launcher=lambda *a: None, + ) + + def test_skip_verify_reaches_the_sync_which_reports_the_skip(self): + warnings = [] + calls = {} + + def fake_sync(command, base_env, base_url, api_key, skip_verify): + calls["skip_verify"] = skip_verify + return ModelSyncSkipped("offline") + + run_agent( + "http://localhost:4000", + "sk-key", + ["opencode"], + skip_verify=True, + base_env={}, + sync_models=fake_sync, + warn=warnings.append, + which=lambda name: "/usr/local/bin/opencode", + verify=lambda *a: pytest.fail("--skip-verify must not verify"), + launcher=lambda p, a, e: calls.update(env=dict(e)), + ) + assert calls["skip_verify"] is True + assert "OPENCODE_CONFIG_CONTENT" not in calls["env"] + assert warnings == ["litellm: not syncing OpenCode models from the proxy: offline"] + + def test_skipped_sync_still_launches_with_plain_openai_env(self): + calls = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["opencode"], + base_env={}, + sync_models=lambda *a: ModelSyncSkipped("proxy said no"), + warn=lambda message: calls.setdefault("warned", message), + which=lambda name: "/usr/local/bin/opencode", + verify=lambda *a: None, + launcher=lambda p, a, e: calls.update(env=dict(e)), + ) + assert calls["env"]["OPENAI_BASE_URL"] == "http://localhost:4000/v1" + assert "OPENCODE_CONFIG_CONTENT" not in calls["env"] + assert "proxy said no" in calls["warned"] + + def test_non_opencode_agent_is_not_warned_about_model_sync(self): + warnings = [] + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + base_env={}, + warn=warnings.append, + which=lambda name: "/usr/local/bin/claude", + verify=lambda *a: None, + launcher=lambda *a: None, + sync_models=agent_model_sync_env, + ) + assert warnings == [] + + def test_default_sync_is_the_agent_model_sync(self): + assert _default_of(run_agent, "sync_models") is agent_model_sync_env + def test_wires_env_and_launches_resolved_binary(self): calls = {} @@ -662,6 +919,19 @@ class TestAgentCommands: assert captured["command"] == ["codex", "exec", "do a thing"] assert "routing Codex through proxy" in result.output + def test_opencode_launches_through_the_proxy(self): + captured = {} + with patch(f"{AGENTS_MODULE}.run_agent", side_effect=lambda b, k, c, **kw: captured.update(command=list(c))): + result = self.runner.invoke( + _agent_command("opencode"), + [], + obj={"base_url": "http://localhost:4000", "api_key": "sk-key"}, + ) + + assert result.exit_code == 0, result.output + assert captured["command"] == ["opencode"] + assert "routing OpenCode through proxy at http://localhost:4000" in result.output + def test_skip_verify_is_consumed_not_forwarded(self): captured = {}