From 1490f2ae3653e4537cd302c6db46396888025839 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 23 Jul 2026 04:50:05 +0000 Subject: [PATCH] feat(autoroute): install proxy runtime on demand so lite autoroute up works on a thin litellm[cli] install Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/client/cli/README.md | 4 +- .../client/cli/commands/autoroute/commands.py | 37 +++-- .../client/cli/commands/autoroute/process.py | 95 ++++++++++++- scripts/install-cli.sh | 6 +- .../client/cli/autoroute/test_commands.py | 70 +++++++++- .../client/cli/autoroute/test_process.py | 131 ++++++++++++++++++ 6 files changed, 321 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 84bc27ef0d4..235c33355e9 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -495,7 +495,7 @@ Cursor is not supported: it has no equivalent file-based config to hot-patch thi #### Install the CLI -`lite autoroute up` builds and runs a throwaway litellm proxy locally, so unlike the rest of this CLI it needs the proxy server runtime, not just the thin `litellm[cli]` client. Install `litellm[proxy]` (which ships the `lite` command too) with a single curl command -- no existing Python tooling required, `uv` is bootstrapped automatically if missing: +`lite autoroute up` builds and runs a throwaway litellm proxy locally, so unlike the rest of this CLI it needs the proxy server runtime, not just the thin `litellm[cli]` client. The quickest way to get everything up front is to install `litellm[proxy]` (which ships the `lite` command too) with a single curl command -- no existing Python tooling required, `uv` is bootstrapped automatically if missing: ```bash curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh @@ -508,7 +508,7 @@ curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm// LITELLM_CLI_REF= sh ``` -The thin `scripts/install-cli.sh` installs only `litellm[cli]`, which is enough for `lite login`, `lite claude`, and `lite up`, but not for `lite autoroute up`; running it against a `litellm[cli]` install fails fast with a message telling you to install the proxy runtime. +The thin `scripts/install-cli.sh` installs only `litellm[cli]`, which is all `lite login`, `lite claude`, and `lite up` need. It works for `lite autoroute up` too: the first `up` on a thin install provisions the proxy runtime on demand, running `uv pip install` for litellm's `proxy` dependencies (read from the installed distribution, so a branch/QA install pulls that branch's own pinned versions) before it launches the local proxy. Installing `litellm[proxy]` up front just front-loads that step so the first `up` starts instantly. Point the CLI at your real proxy and key before running any `lite model-groups` or `lite autoroute` command -- like every other command in this CLI, they read `LITELLM_PROXY_URL`/`LITELLM_PROXY_API_KEY` (or `--base-url`/`--api-key`), no `lite login` required: diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 381a99f453e..ccaf6ad151a 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -19,7 +19,9 @@ from .process import ( LOG_PATH, PidRecord, ProcessLaunchError, + ProxyRuntimeInstallError, clear_pid_record, + install_proxy_runtime, is_port_available, is_running, launch_proxy, @@ -70,6 +72,31 @@ def _ensure_master_key() -> str: return master_key +def _ensure_proxy_runtime() -> None: + """Provision the proxy-server runtime that ``up`` needs but the thin ``litellm[cli]`` install lacks. + + A no-op once the runtime is present, so repeat ``up`` runs pay nothing. + """ + missing = missing_proxy_runtime_modules() + if not missing: + return + click.echo( + "lite autoroute up launches a local litellm proxy, which needs the proxy runtime that the " + f"thin litellm[cli] install omits (missing: {', '.join(missing)}). Installing it now with uv..." + ) + try: + install_proxy_runtime() + except ProxyRuntimeInstallError as e: + raise click.ClickException(str(e)) + still_missing = missing_proxy_runtime_modules() + if still_missing: + raise click.ClickException( + "Installed the litellm proxy runtime but these modules are still missing: " + f"{', '.join(still_missing)}. Install it manually with `uv tool install --force 'litellm[proxy]'`." + ) + click.echo("Proxy runtime installed.") + + @click.group(name="autoroute") def autoroute_group() -> None: """QA complexity-based auto-routing against models your key can already use""" @@ -95,15 +122,7 @@ def up(port: int) -> None: if not CONFIG_PATH.exists(): raise click.ClickException("No config found. Run `lite autoroute configure` first.") - missing = missing_proxy_runtime_modules() - if missing: - raise click.ClickException( - "lite autoroute up launches a local litellm proxy, which needs the proxy runtime that the " - f"thin `litellm[cli]` install does not include (missing: {', '.join(missing)}). Install the " - "proxy runtime with `uv tool install --force 'litellm[proxy]'`, or to QA a branch, " - "`curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm//scripts/install.sh | " - "LITELLM_CLI_REF= sh`." - ) + _ensure_proxy_runtime() try: existing_pid = read_pid_record() diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py index 5a7f016186a..3a2cdef01fa 100644 --- a/litellm/proxy/client/cli/commands/autoroute/process.py +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -1,18 +1,23 @@ import contextlib +import importlib import importlib.util import json import os +import shutil import signal import socket import subprocess import sys import threading import time +from collections.abc import Callable from dataclasses import dataclass +from importlib import metadata from pathlib import Path import click import requests +from packaging.requirements import Requirement from pydantic import TypeAdapter, ValidationError from ..up import UpError, secure_create @@ -46,12 +51,93 @@ def missing_proxy_runtime_modules() -> tuple[str, ...]: ``launch_proxy`` runs the full ``litellm.proxy.proxy_cli`` server, whose dependencies live in the ``proxy`` extra, not the ``cli`` extra that installs the ``lite`` command. On a thin - ``litellm[cli]`` install the subprocess dies with a bare ``ModuleNotFoundError``; detecting the - gap here lets ``up`` fail with an actionable message instead. + ``litellm[cli]`` install the subprocess would die with a bare ``ModuleNotFoundError``; detecting + the gap here lets ``up`` install the runtime on demand (see ``install_proxy_runtime``). """ return tuple(name for name in _PROXY_RUNTIME_MODULES if importlib.util.find_spec(name) is None) +class ProxyRuntimeInstallError(Exception): + """Raised when the litellm proxy runtime cannot be installed on demand for ``lite autoroute up``.""" + + +@dataclass(frozen=True, slots=True) +class CommandResult: + returncode: int + output: str + + +def proxy_extra_requirements() -> tuple[str, ...]: + """The dependency specifiers behind litellm's ``proxy`` extra, read from the installed distribution. + + Reading them from metadata (rather than hardcoding them) keeps them in lockstep with the + running litellm: a branch/QA install via ``LITELLM_CLI_REF`` reports that branch's own proxy + dependencies at their pinned versions. Only extra-gated requirements are returned and their + markers are dropped, so the ``sys_platform != 'win32'`` guards on a couple of them resolve to + install on the macOS/Linux hosts the ``lite`` installer supports. + """ + raw_requirements = metadata.metadata("litellm").get_all("Requires-Dist") or () + parsed = tuple(Requirement(str(raw)) for raw in raw_requirements) + return tuple( + f"{req.name}{req.specifier}" + for req in parsed + if req.marker is not None and req.marker.evaluate({"extra": "proxy"}) and not req.marker.evaluate({"extra": ""}) + ) + + +def find_uv() -> str | None: + """Locate the uv executable, falling back to the path its official installer writes to.""" + on_path = shutil.which("uv") + if on_path is not None: + return on_path + fallback = Path.home() / ".local" / "bin" / "uv" + return str(fallback) if os.access(fallback, os.X_OK) else None + + +def _run_uv_install(uv: str, requirements: tuple[str, ...]) -> CommandResult: + completed = subprocess.run( + [uv, "pip", "install", "--python", sys.executable, *requirements], + capture_output=True, + text=True, + ) + return CommandResult( + returncode=completed.returncode, + output=completed.stderr.strip() or completed.stdout.strip(), + ) + + +def install_proxy_runtime( + find_uv_bin: Callable[[], str | None] = find_uv, + requirements: Callable[[], tuple[str, ...]] = proxy_extra_requirements, + run_install: Callable[[str, tuple[str, ...]], CommandResult] = _run_uv_install, +) -> None: + """Install litellm's ``proxy`` extra into the running interpreter so ``up`` can launch the server. + + The thin ``litellm[cli]`` install omits the proxy-server runtime, so ``up`` provisions it on + first use. litellm itself is deliberately excluded from the install set so its already-installed + version (release or QA branch) is never swapped out from under the running command. + """ + uv = find_uv_bin() + if uv is None: + raise ProxyRuntimeInstallError( + "Could not find uv to install the proxy runtime. Install it manually with " + "`uv tool install --force 'litellm[proxy]'`." + ) + specifiers = requirements() + if not specifiers: + raise ProxyRuntimeInstallError( + "Could not read litellm's proxy dependencies from its installed metadata. Install the " + "runtime manually with `uv tool install --force 'litellm[proxy]'`." + ) + result = run_install(uv, specifiers) + if result.returncode != 0: + raise ProxyRuntimeInstallError( + f"Installing the proxy runtime failed:\n{result.output}\n" + "Install it manually with `uv tool install --force 'litellm[proxy]'`." + ) + importlib.invalidate_caches() + + DEFAULT_AUTOROUTE_PORT = 5483 @@ -182,13 +268,18 @@ __all__ = [ "DEFAULT_AUTOROUTE_PORT", "LOG_PATH", "PID_RECORD_PATH", + "CommandResult", "PidRecord", "ProcessLaunchError", + "ProxyRuntimeInstallError", "clear_pid_record", + "find_uv", + "install_proxy_runtime", "is_port_available", "is_running", "launch_proxy", "missing_proxy_runtime_modules", + "proxy_extra_requirements", "poll_liveliness", "read_pid_record", "secure_create", diff --git a/scripts/install-cli.sh b/scripts/install-cli.sh index a39b73c2e5a..9281dec2970 100755 --- a/scripts/install-cli.sh +++ b/scripts/install-cli.sh @@ -4,8 +4,10 @@ # # Installs only litellm[cli]: the `lite` command for authenticating to a LiteLLM # proxy and running coding agents (lite claude / codex / opencode) through it. -# None of the proxy server runtime is pulled in. To run a proxy server instead, -# use scripts/install.sh, which installs litellm[proxy]. +# None of the proxy server runtime is pulled in. `lite autoroute up` still works +# on this thin install -- it provisions the proxy runtime on demand the first time +# it runs. To pull the full proxy server up front instead, use scripts/install.sh, +# which installs litellm[proxy]. # # Needs only curl: uv is bootstrapped if missing, and uv provisions a compatible # Python itself (honouring litellm's requires-python), downloading a managed one diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 74bf1c95777..b5aee9162f9 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -71,24 +71,80 @@ class TestUpCommand: assert result.exception is None or isinstance(result.exception, SystemExit) assert "lite autoroute configure" in result.output - def test_refuses_with_actionable_error_when_proxy_runtime_missing(self, monkeypatch, tmp_path): + def test_installs_the_proxy_runtime_on_demand_then_launches(self, monkeypatch, tmp_path): """`up` launches a real litellm proxy, which the thin `litellm[cli]` install cannot run. - It must fail fast with an actionable message pointing at the proxy install, before it ever - tries to launch the doomed subprocess (which would otherwise die with a bare ImportError).""" + Rather than refusing, it provisions the proxy runtime on demand and then proceeds to launch + -- so a thin install ends up working without the user re-running a different installer.""" + config_path, _log_path, claude_settings_path, _backup_path, _pid_record_path = _patch_paths( + monkeypatch, tmp_path + ) + config_path.write_text(yaml.safe_dump({"model_list": []})) + claude_settings_path.write_text(json.dumps({"theme": "dark"})) + _silence_signal_handling(monkeypatch) + + check_count = {"n": 0} + + def _missing(): + check_count["n"] += 1 + return ("fastapi", "websockets") if check_count["n"] == 1 else () + + install_calls = [] + launched = [] + monkeypatch.setattr(commands_module, "missing_proxy_runtime_modules", _missing) + monkeypatch.setattr(commands_module, "install_proxy_runtime", lambda: install_calls.append(True)) + monkeypatch.setattr(commands_module, "launch_proxy", lambda *a, **k: launched.append(True) or FakeProcess(88)) + monkeypatch.setattr(commands_module, "poll_liveliness", lambda *a, **k: None) + monkeypatch.setattr(commands_module, "is_port_available", lambda port: True) + monkeypatch.setattr(commands_module, "terminate", lambda pid, **k: None) + monkeypatch.setattr(commands_module.secrets, "token_urlsafe", lambda n: "fixed-master-key") + monkeypatch.setattr("threading.Event.wait", lambda self, timeout=None: True) + + result = self.runner.invoke(up) + + assert result.exit_code == 0, result.output + assert install_calls == [True] + assert launched == [True] + assert "fastapi, websockets" in result.output + + def test_surfaces_error_and_does_not_launch_when_runtime_install_fails(self, monkeypatch, tmp_path): + """If provisioning the proxy runtime fails, `up` must surface the reason and never launch the + doomed proxy subprocess (which would otherwise die with a bare ImportError).""" config_path, _log_path, _settings_path, _backup_path, _pid_record_path = _patch_paths(monkeypatch, tmp_path) config_path.write_text(yaml.safe_dump({"model_list": []})) - monkeypatch.setattr(commands_module, "missing_proxy_runtime_modules", lambda: ("fastapi", "websockets")) + monkeypatch.setattr(commands_module, "missing_proxy_runtime_modules", lambda: ("fastapi",)) + + def _boom(): + raise commands_module.ProxyRuntimeInstallError("uv could not reach the index: network is down") def _fail_if_launched(*args, **kwargs): - raise AssertionError("launch_proxy must not run when the proxy runtime is missing") + raise AssertionError("launch_proxy must not run when the runtime install failed") + + monkeypatch.setattr(commands_module, "install_proxy_runtime", _boom) + monkeypatch.setattr(commands_module, "launch_proxy", _fail_if_launched) + + result = self.runner.invoke(up) + + assert result.exit_code != 0 + assert "network is down" in result.output + + def test_errors_when_runtime_is_still_missing_after_install(self, monkeypatch, tmp_path): + """A `install_proxy_runtime` that reports success but leaves modules still unimportable must + not fall through to launching the proxy; `up` reports the still-missing modules instead.""" + config_path, _log_path, _settings_path, _backup_path, _pid_record_path = _patch_paths(monkeypatch, tmp_path) + config_path.write_text(yaml.safe_dump({"model_list": []})) + monkeypatch.setattr(commands_module, "missing_proxy_runtime_modules", lambda: ("fastapi",)) + monkeypatch.setattr(commands_module, "install_proxy_runtime", lambda: None) + + def _fail_if_launched(*args, **kwargs): + raise AssertionError("launch_proxy must not run while the runtime is still missing") monkeypatch.setattr(commands_module, "launch_proxy", _fail_if_launched) result = self.runner.invoke(up) assert result.exit_code != 0 - assert "fastapi, websockets" in result.output - assert "litellm[proxy]" in result.output + assert "still missing" in result.output + assert "fastapi" in result.output def test_refuses_when_pid_record_exists_and_process_still_running(self, monkeypatch, tmp_path): config_path, _log_path, _settings_path, _backup_path, pid_record_path = _patch_paths(monkeypatch, tmp_path) diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_process.py b/tests/test_litellm/proxy/client/cli/autoroute/test_process.py index 478b64c2d78..f67b23a5e87 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_process.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_process.py @@ -4,18 +4,24 @@ from typing import Optional from unittest.mock import patch import pytest +from packaging.requirements import Requirement from litellm.proxy.client.cli.commands.autoroute import process as process_module from litellm.proxy.client.cli.commands.autoroute.process import ( + CommandResult, PidRecord, ProcessLaunchError, + ProxyRuntimeInstallError, UpError, clear_pid_record, + find_uv, + install_proxy_runtime, is_port_available, is_running, launch_proxy, missing_proxy_runtime_modules, poll_liveliness, + proxy_extra_requirements, read_pid_record, write_pid_record, ) @@ -165,3 +171,128 @@ class TestMissingProxyRuntimeModules: monkeypatch.setattr(process_module, "_PROXY_RUNTIME_MODULES", ("os", "socket")) assert missing_proxy_runtime_modules() == () + + +class TestProxyExtraRequirements: + def test_returns_pinned_proxy_deps_without_litellm_or_markers(self): + """`up` installs these on demand, so they must be litellm's real proxy dependencies with + their version pins intact, must never include litellm itself (that would swap the running + install), and must carry no environment markers (they are dropped so the win32-guarded + entries still install on the macOS/Linux hosts the CLI supports).""" + reqs = proxy_extra_requirements() + + names = {Requirement(r).name for r in reqs} + assert {"fastapi", "uvicorn", "apscheduler", "backoff", "orjson", "websockets"} <= names + assert "litellm" not in names + assert all(str(Requirement(r).specifier) for r in reqs) + assert all(";" not in r for r in reqs) + + +class TestFindUv: + def test_prefers_uv_on_path(self, monkeypatch): + monkeypatch.setattr(process_module.shutil, "which", lambda name: "/usr/bin/uv" if name == "uv" else None) + + assert find_uv() == "/usr/bin/uv" + + def test_falls_back_to_the_installers_local_bin_path(self, monkeypatch, tmp_path): + monkeypatch.setattr(process_module.shutil, "which", lambda name: None) + uv_path = tmp_path / ".local" / "bin" / "uv" + uv_path.parent.mkdir(parents=True) + uv_path.write_text("#!/bin/sh\n") + uv_path.chmod(0o755) + monkeypatch.setattr(process_module.Path, "home", lambda: tmp_path) + + assert find_uv() == str(uv_path) + + def test_returns_none_when_uv_absent_everywhere(self, monkeypatch, tmp_path): + monkeypatch.setattr(process_module.shutil, "which", lambda name: None) + monkeypatch.setattr(process_module.Path, "home", lambda: tmp_path) + + assert find_uv() is None + + +class TestRunUvInstall: + def test_targets_the_current_interpreter_and_never_reinstalls_litellm(self, monkeypatch): + """The install must land in the interpreter running `lite` (so the proxy subprocess sees it) + and pass only the extra's deps -- passing `litellm` would let uv resolve a different version + over the installed (possibly branch/QA) one.""" + captured = {} + + class _Completed: + returncode = 0 + stdout = "ok" + stderr = "" + + def _fake_run(argv, capture_output, text): + captured["argv"] = argv + return _Completed() + + monkeypatch.setattr(process_module.subprocess, "run", _fake_run) + + result = process_module._run_uv_install("/bin/uv", ("fastapi>=1", "uvicorn>=1")) + + assert result == CommandResult(returncode=0, output="ok") + assert captured["argv"] == [ + "/bin/uv", + "pip", + "install", + "--python", + process_module.sys.executable, + "fastapi>=1", + "uvicorn>=1", + ] + assert "litellm" not in captured["argv"][5:] + + def test_prefers_stderr_when_the_command_fails(self, monkeypatch): + class _Completed: + returncode = 1 + stdout = "some stdout" + stderr = "the real error" + + monkeypatch.setattr(process_module.subprocess, "run", lambda argv, capture_output, text: _Completed()) + + assert process_module._run_uv_install("/bin/uv", ("fastapi>=1",)) == CommandResult( + returncode=1, output="the real error" + ) + + +class TestInstallProxyRuntime: + def test_raises_when_uv_is_missing(self): + with pytest.raises(ProxyRuntimeInstallError) as excinfo: + install_proxy_runtime(find_uv_bin=lambda: None) + + assert "litellm[proxy]" in str(excinfo.value) + + def test_raises_when_no_proxy_requirements_are_found(self): + with pytest.raises(ProxyRuntimeInstallError) as excinfo: + install_proxy_runtime(find_uv_bin=lambda: "/bin/uv", requirements=lambda: ()) + + assert "metadata" in str(excinfo.value) + + def test_raises_with_the_command_output_when_install_fails(self): + def _run(uv, reqs): + return CommandResult(returncode=1, output="network is unreachable") + + with pytest.raises(ProxyRuntimeInstallError) as excinfo: + install_proxy_runtime( + find_uv_bin=lambda: "/bin/uv", + requirements=lambda: ("fastapi>=1",), + run_install=_run, + ) + + assert "network is unreachable" in str(excinfo.value) + + def test_installs_the_discovered_requirements_with_the_found_uv(self): + calls = [] + + def _run(uv, reqs): + calls.append((uv, reqs)) + return CommandResult(returncode=0, output="") + + install_proxy_runtime( + find_uv_bin=lambda: "/bin/uv", + requirements=lambda: ("fastapi>=1", "uvicorn>=1"), + run_install=_run, + ) + + assert calls == [("/bin/uv", ("fastapi>=1", "uvicorn>=1"))]