From 4a8563b01b2a9f086836f839883cb07ed13521b4 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 10:24:24 +0900 Subject: [PATCH] feat(proxy): opt into Rust forwarding with LITELLM_RUST One switch turns on every optional Rust path, so forwarding reads the same variable through the same parser as the rust bridge instead of its own LITELLM_RUST_GATEWAY. An unrecognized value now leaves forwarding off, like every other Rust path, instead of failing startup. Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/rust_gateway.py | 11 +++---- litellm/rust_bridge/configuration.py | 9 ++++-- .../configuration/test_rust_gateway.py | 4 +-- tests/unit/gateway/test_launch.py | 2 +- tests/unit/proxy/test_rust_gateway.py | 30 ++++++++----------- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/rust_gateway.py b/litellm/proxy/rust_gateway.py index 0dbf3a4de3d..8be060e362c 100644 --- a/litellm/proxy/rust_gateway.py +++ b/litellm/proxy/rust_gateway.py @@ -14,10 +14,11 @@ from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.rust_bridge.configuration import ENV_NAME, environment_switch + if TYPE_CHECKING: from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope -ENABLE: Final = "LITELLM_RUST_GATEWAY" GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD" METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST" PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS" @@ -61,7 +62,7 @@ def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode: _ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")), environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES), ) - if _ENABLED.validate_python(environ.get(ENABLE, "false")): + if environment_switch(environ): return Supervisor() return Disabled() @@ -109,11 +110,11 @@ def gateway_binary(scripts_dir: str) -> str | None: def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None: flag: Final = next((name for name, value in unsupported.items() if value), None) if flag is not None: - return f"{ENABLE} does not support {flag}" + return f"{ENV_NAME} does not support {flag}" if os.name != "posix": - return f"{ENABLE} requires POSIX inherited sockets and process signals" + return f"{ENV_NAME} requires POSIX inherited sockets and process signals" if binary is None: - return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment" + return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment" return None diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 152ba632996..e36527e9f4e 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from enum import Enum, auto from functools import lru_cache from typing import Final @@ -8,7 +9,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never -_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +ENV_NAME: Final = "LITELLM_RUST" _ENV_BOOL: Final = TypeAdapter(bool) @@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None: return None +def environment_switch(environ: Mapping[str, str]) -> bool | None: + return _parse_env_bool(environ.get(ENV_NAME)) + + def decide( rollout: Rollout, *, @@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision: return decide( rollout, process_override=_CONFIGURATION.override, - environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), + environment_override=environment_switch(os.environ), ) diff --git a/tests/integration/configuration/test_rust_gateway.py b/tests/integration/configuration/test_rust_gateway.py index 06d77d6f3d0..b75da286bf7 100644 --- a/tests/integration/configuration/test_rust_gateway.py +++ b/tests/integration/configuration/test_rust_gateway.py @@ -55,7 +55,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip( "LITELLM_MASTER_KEY": gateway.key, "STORE_MODEL_IN_DB": "False", "PYTHON_DOTENV_DISABLED": "1", - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "FORWARDED_ALLOW_IPS": TRUSTED_BALANCER, } with (tmp_path / "proxy.log").open("wb") as output: @@ -124,7 +124,7 @@ def test_gateway_launcher_private_hop_preserves_client_address( environment: Final = { **{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"}, "LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret", - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "LITELLM_PGBOUNCER_ENABLED": "false", "CONFIG_FILE_PATH": str(config), "STORE_MODEL_IN_DB": "False", diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index 789067653b7..418e0358b64 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -280,7 +280,7 @@ class TestMain: def test_forwarding_disabled_keeps_uvicorn_socket_options( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false") + monkeypatch.setenv("LITELLM_RUST", "false") monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) served: Final[list[tuple[str, ...]]] = [] diff --git a/tests/unit/proxy/test_rust_gateway.py b/tests/unit/proxy/test_rust_gateway.py index b71a098ddb3..1072148be9f 100644 --- a/tests/unit/proxy/test_rust_gateway.py +++ b/tests/unit/proxy/test_rust_gateway.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Final import pytest from click.testing import CliRunner -from pydantic import ValidationError from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware if TYPE_CHECKING: @@ -52,28 +51,30 @@ INT: Final = signal.SIGINT "flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"] ) def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true") + monkeypatch.setenv("LITELLM_RUST", "true") monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False) values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else [] result: Final = CliRunner().invoke(run_server, [flag, *values]) assert result.exit_code != 0 - assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output + assert f"LITELLM_RUST does not support {flag}" in result.output @pytest.mark.parametrize( ("environ", "expected"), [ ({}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "false"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "off"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "0"}, Disabled()), - ({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()), - ({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()), + ({"LITELLM_RUST": "false"}, Disabled()), + ({"LITELLM_RUST": "off"}, Disabled()), + ({"LITELLM_RUST": "0"}, Disabled()), + ({"LITELLM_RUST": "true"}, Supervisor()), + ({"LITELLM_RUST": "1"}, Supervisor()), + ({"LITELLM_RUST": "yes"}, Supervisor()), + ({"LITELLM_RUST": "on"}, Supervisor()), + ({"LITELLM_RUST": " 1 "}, Supervisor()), + ({"LITELLM_RUST": "enabled"}, Disabled()), ( { - "LITELLM_RUST_GATEWAY": "true", + "LITELLM_RUST": "true", "_LITELLM_GATEWAY_FD": "17", "_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0", "FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5", @@ -86,11 +87,6 @@ def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: ob assert rust_gateway_mode(environ) == expected -def test_unrecognized_switch_value_is_rejected() -> None: - with pytest.raises(ValidationError): - rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"}) - - @pytest.mark.parametrize( ("environ", "expected"), [ @@ -131,7 +127,7 @@ def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> Non def test_missing_binary_gives_install_instruction() -> None: assert startup_error(None, {}) == ( - "LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment" + "LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment" )