feat(proxy): opt into Rust forwarding with LITELLM_RUST

One switch turns on every optional Rust path, so forwarding reads the same
variable through the same parser as the rust bridge instead of its own
LITELLM_RUST_GATEWAY. An unrecognized value now leaves forwarding off, like
every other Rust path, instead of failing startup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-11-03 10:24:24 +09:00
parent 543c015305
commit 4a8563b01b
5 changed files with 29 additions and 27 deletions

View file

@ -14,10 +14,11 @@ from typing import TYPE_CHECKING, Final, TypeAlias
from pydantic import TypeAdapter
from typing_extensions import assert_never
from litellm.rust_bridge.configuration import ENV_NAME, environment_switch
if TYPE_CHECKING:
from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
ENABLE: Final = "LITELLM_RUST_GATEWAY"
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
@ -61,7 +62,7 @@ def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode:
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
)
if _ENABLED.validate_python(environ.get(ENABLE, "false")):
if environment_switch(environ):
return Supervisor()
return Disabled()
@ -109,11 +110,11 @@ def gateway_binary(scripts_dir: str) -> str | None:
def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None:
flag: Final = next((name for name, value in unsupported.items() if value), None)
if flag is not None:
return f"{ENABLE} does not support {flag}"
return f"{ENV_NAME} does not support {flag}"
if os.name != "posix":
return f"{ENABLE} requires POSIX inherited sockets and process signals"
return f"{ENV_NAME} requires POSIX inherited sockets and process signals"
if binary is None:
return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
return None

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from enum import Enum, auto
from functools import lru_cache
from typing import Final
@ -8,7 +9,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
_GLOBAL_ENV_NAME: Final = "LITELLM_RUST"
ENV_NAME: Final = "LITELLM_RUST"
_ENV_BOOL: Final = TypeAdapter(bool)
@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None:
return None
def environment_switch(environ: Mapping[str, str]) -> bool | None:
return _parse_env_bool(environ.get(ENV_NAME))
def decide(
rollout: Rollout,
*,
@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision:
return decide(
rollout,
process_override=_CONFIGURATION.override,
environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)),
environment_override=environment_switch(os.environ),
)

View file

@ -55,7 +55,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
"LITELLM_MASTER_KEY": gateway.key,
"STORE_MODEL_IN_DB": "False",
"PYTHON_DOTENV_DISABLED": "1",
"LITELLM_RUST_GATEWAY": "true",
"LITELLM_RUST": "true",
"FORWARDED_ALLOW_IPS": TRUSTED_BALANCER,
}
with (tmp_path / "proxy.log").open("wb") as output:
@ -124,7 +124,7 @@ def test_gateway_launcher_private_hop_preserves_client_address(
environment: Final = {
**{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"},
"LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret",
"LITELLM_RUST_GATEWAY": "true",
"LITELLM_RUST": "true",
"LITELLM_PGBOUNCER_ENABLED": "false",
"CONFIG_FILE_PATH": str(config),
"STORE_MODEL_IN_DB": "False",

View file

@ -280,7 +280,7 @@ class TestMain:
def test_forwarding_disabled_keeps_uvicorn_socket_options(
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false")
monkeypatch.setenv("LITELLM_RUST", "false")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
served: Final[list[tuple[str, ...]]] = []

View file

@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Final
import pytest
from click.testing import CliRunner
from pydantic import ValidationError
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
if TYPE_CHECKING:
@ -52,28 +51,30 @@ INT: Final = signal.SIGINT
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
)
def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true")
monkeypatch.setenv("LITELLM_RUST", "true")
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
result: Final = CliRunner().invoke(run_server, [flag, *values])
assert result.exit_code != 0
assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output
assert f"LITELLM_RUST does not support {flag}" in result.output
@pytest.mark.parametrize(
("environ", "expected"),
[
({}, Disabled()),
({"LITELLM_RUST_GATEWAY": "false"}, Disabled()),
({"LITELLM_RUST_GATEWAY": "off"}, Disabled()),
({"LITELLM_RUST_GATEWAY": "0"}, Disabled()),
({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()),
({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()),
({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()),
({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()),
({"LITELLM_RUST": "false"}, Disabled()),
({"LITELLM_RUST": "off"}, Disabled()),
({"LITELLM_RUST": "0"}, Disabled()),
({"LITELLM_RUST": "true"}, Supervisor()),
({"LITELLM_RUST": "1"}, Supervisor()),
({"LITELLM_RUST": "yes"}, Supervisor()),
({"LITELLM_RUST": "on"}, Supervisor()),
({"LITELLM_RUST": " 1 "}, Supervisor()),
({"LITELLM_RUST": "enabled"}, Disabled()),
(
{
"LITELLM_RUST_GATEWAY": "true",
"LITELLM_RUST": "true",
"_LITELLM_GATEWAY_FD": "17",
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
@ -86,11 +87,6 @@ def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: ob
assert rust_gateway_mode(environ) == expected
def test_unrecognized_switch_value_is_rejected() -> None:
with pytest.raises(ValidationError):
rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"})
@pytest.mark.parametrize(
("environ", "expected"),
[
@ -131,7 +127,7 @@ def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> Non
def test_missing_binary_gives_install_instruction() -> None:
assert startup_error(None, {}) == (
"LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment"
"LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment"
)