mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(proxy): opt into Rust forwarding with LITELLM_RUST
One switch turns on every optional Rust path, so forwarding reads the same variable through the same parser as the rust bridge instead of its own LITELLM_RUST_GATEWAY. An unrecognized value now leaves forwarding off, like every other Rust path, instead of failing startup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
543c015305
commit
4a8563b01b
5 changed files with 29 additions and 27 deletions
|
|
@ -14,10 +14,11 @@ from typing import TYPE_CHECKING, Final, TypeAlias
|
|||
from pydantic import TypeAdapter
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.rust_bridge.configuration import ENV_NAME, environment_switch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
|
||||
|
||||
ENABLE: Final = "LITELLM_RUST_GATEWAY"
|
||||
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
|
||||
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
|
||||
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
|
||||
|
|
@ -61,7 +62,7 @@ def rust_gateway_mode(environ: Mapping[str, str]) -> RustGatewayMode:
|
|||
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
|
||||
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
|
||||
)
|
||||
if _ENABLED.validate_python(environ.get(ENABLE, "false")):
|
||||
if environment_switch(environ):
|
||||
return Supervisor()
|
||||
return Disabled()
|
||||
|
||||
|
|
@ -109,11 +110,11 @@ def gateway_binary(scripts_dir: str) -> str | None:
|
|||
def startup_error(binary: str | None, unsupported: Mapping[str, object]) -> str | None:
|
||||
flag: Final = next((name for name, value in unsupported.items() if value), None)
|
||||
if flag is not None:
|
||||
return f"{ENABLE} does not support {flag}"
|
||||
return f"{ENV_NAME} does not support {flag}"
|
||||
if os.name != "posix":
|
||||
return f"{ENABLE} requires POSIX inherited sockets and process signals"
|
||||
return f"{ENV_NAME} requires POSIX inherited sockets and process signals"
|
||||
if binary is None:
|
||||
return f"{ENABLE} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
return f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from enum import Enum, auto
|
||||
from functools import lru_cache
|
||||
from typing import Final
|
||||
|
|
@ -8,7 +9,7 @@ from typing import Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never
|
||||
|
||||
_GLOBAL_ENV_NAME: Final = "LITELLM_RUST"
|
||||
ENV_NAME: Final = "LITELLM_RUST"
|
||||
_ENV_BOOL: Final = TypeAdapter(bool)
|
||||
|
||||
|
||||
|
|
@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None:
|
|||
return None
|
||||
|
||||
|
||||
def environment_switch(environ: Mapping[str, str]) -> bool | None:
|
||||
return _parse_env_bool(environ.get(ENV_NAME))
|
||||
|
||||
|
||||
def decide(
|
||||
rollout: Rollout,
|
||||
*,
|
||||
|
|
@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision:
|
|||
return decide(
|
||||
rollout,
|
||||
process_override=_CONFIGURATION.override,
|
||||
environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)),
|
||||
environment_override=environment_switch(os.environ),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
|
|||
"LITELLM_MASTER_KEY": gateway.key,
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
"PYTHON_DOTENV_DISABLED": "1",
|
||||
"LITELLM_RUST_GATEWAY": "true",
|
||||
"LITELLM_RUST": "true",
|
||||
"FORWARDED_ALLOW_IPS": TRUSTED_BALANCER,
|
||||
}
|
||||
with (tmp_path / "proxy.log").open("wb") as output:
|
||||
|
|
@ -124,7 +124,7 @@ def test_gateway_launcher_private_hop_preserves_client_address(
|
|||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"},
|
||||
"LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret",
|
||||
"LITELLM_RUST_GATEWAY": "true",
|
||||
"LITELLM_RUST": "true",
|
||||
"LITELLM_PGBOUNCER_ENABLED": "false",
|
||||
"CONFIG_FILE_PATH": str(config),
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ class TestMain:
|
|||
def test_forwarding_disabled_keeps_uvicorn_socket_options(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "false")
|
||||
monkeypatch.setenv("LITELLM_RUST", "false")
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
|
||||
served: Final[list[tuple[str, ...]]] = []
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
from pydantic import ValidationError
|
||||
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -52,28 +51,30 @@ INT: Final = signal.SIGINT
|
|||
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
|
||||
)
|
||||
def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST_GATEWAY", "true")
|
||||
monkeypatch.setenv("LITELLM_RUST", "true")
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
|
||||
result: Final = CliRunner().invoke(run_server, [flag, *values])
|
||||
assert result.exit_code != 0
|
||||
assert f"LITELLM_RUST_GATEWAY does not support {flag}" in result.output
|
||||
assert f"LITELLM_RUST does not support {flag}" in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("environ", "expected"),
|
||||
[
|
||||
({}, Disabled()),
|
||||
({"LITELLM_RUST_GATEWAY": "false"}, Disabled()),
|
||||
({"LITELLM_RUST_GATEWAY": "off"}, Disabled()),
|
||||
({"LITELLM_RUST_GATEWAY": "0"}, Disabled()),
|
||||
({"LITELLM_RUST_GATEWAY": "true"}, Supervisor()),
|
||||
({"LITELLM_RUST_GATEWAY": "1"}, Supervisor()),
|
||||
({"LITELLM_RUST_GATEWAY": "yes"}, Supervisor()),
|
||||
({"LITELLM_RUST_GATEWAY": "on"}, Supervisor()),
|
||||
({"LITELLM_RUST": "false"}, Disabled()),
|
||||
({"LITELLM_RUST": "off"}, Disabled()),
|
||||
({"LITELLM_RUST": "0"}, Disabled()),
|
||||
({"LITELLM_RUST": "true"}, Supervisor()),
|
||||
({"LITELLM_RUST": "1"}, Supervisor()),
|
||||
({"LITELLM_RUST": "yes"}, Supervisor()),
|
||||
({"LITELLM_RUST": "on"}, Supervisor()),
|
||||
({"LITELLM_RUST": " 1 "}, Supervisor()),
|
||||
({"LITELLM_RUST": "enabled"}, Disabled()),
|
||||
(
|
||||
{
|
||||
"LITELLM_RUST_GATEWAY": "true",
|
||||
"LITELLM_RUST": "true",
|
||||
"_LITELLM_GATEWAY_FD": "17",
|
||||
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
|
||||
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
|
||||
|
|
@ -86,11 +87,6 @@ def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: ob
|
|||
assert rust_gateway_mode(environ) == expected
|
||||
|
||||
|
||||
def test_unrecognized_switch_value_is_rejected() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
rust_gateway_mode({"LITELLM_RUST_GATEWAY": "enabled"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("environ", "expected"),
|
||||
[
|
||||
|
|
@ -131,7 +127,7 @@ def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> Non
|
|||
|
||||
def test_missing_binary_gives_install_instruction() -> None:
|
||||
assert startup_error(None, {}) == (
|
||||
"LITELLM_RUST_GATEWAY requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
"LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue