refactor(auth): decide route rules in Rust (#51531)

Whether a route may be called was decided by Python helpers spread over
RouteChecks and the managed agent module: wildcard, prefix and deny matching,
pass-through grants, and the managed agent route table. The Rust auth core
needs the same rules, and a second copy would drift.

The rules move into a new gateway-auth-core crate under routes/, and Python
calls them through _native. RouteChecks and managed_authorization keep their
function names and now delegate, so every existing caller is unchanged. The
LiteLLMRoutes groups stay Python's and reach Rust through
litellm/rust_bridge/host/route_groups.py.

Route templates parse in a new litellm-starlette crate, which follows
starlette's compile_path and never fails. A pass-through path with an int,
float or uuid parameter used to panic the route group load and now matches
one segment, as the Python matcher did. Both matchers share one walk over
the template that keeps the set of offsets reached so far, so a template
with many path parameters answers in polynomial time where trying every
split, as Python's re does, never finishes. Route lists from key and team
metadata, and JWT team_allowed_routes, read as litellm-config's RouteList, so
a value that is not a list of strings answers with a 403 instead of failing
the request: a malformed allow list grants nothing and a malformed deny list
denies every pass-through route. The Rust route groups are rebuilt whenever
LiteLLMRoutes changes, since an auth-enforced pass-through endpoint adds its
path to openai_routes while the proxy runs. Reading config
files and their include moves out of litellm-config into a new
litellm-config-load crate, which leaves litellm-config the typed data model.

How Python reads a config value it never validates now lives in one place,
python-compat's lenient module: `value is True`, `bool(value)`, and a value of
the expected shape or Malformed. Config types use them as serde_with adapters,
so the gateway's YAML load and python-bridge's from_py read a value the same
way. general_settings.pass_through_endpoints is typed with them, its path as a
starlette RouteTemplate. The config types derive Debug, and veil's Redact where
a field can carry a credential, in place of their hand-written impls.

No change to which well-formed routes are allowed or denied is intended.
This commit is contained in:
yujonglee 2026-11-16 18:55:32 +09:00 • committed by GitHub
parent ad18efa84b
commit 4aa00352bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 2340 additions and 490 deletions

View file

@ -1,79 +1,16 @@
from collections.abc import Mapping
from itertools import product
from types import MappingProxyType
from typing import Annotated, Final, Literal
from pydantic import Field, TypeAdapter, ValidationError
from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore
from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failure
from litellm.rust_bridge._native import auth_managed_agent_route_allowed, auth_managed_route
from litellm.types.agents import AgentResponse
from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext
_MANAGED_REALTIME_ROUTES: Final = frozenset(
(
"/realtime",
"/v1/realtime",
"/openai/v1/realtime",
"/realtime/translations",
"/v1/realtime/translations",
)
)
_MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions"))
_MANAGED_MODEL_ROUTES: Final = frozenset(
f"{prefix}/{operation}"
for prefix, operation in product(
("", "/v1"),
(
"chat/completions",
"completions",
"embeddings",
"responses",
"messages",
"messages/count_tokens",
"images/generations",
"images/edits",
"audio/transcriptions",
"audio/speech",
"moderations",
"rerank",
"decisions",
"systemone",
"extraction",
"ocr",
),
)
) | frozenset(
(
"/openai/v1/responses",
"/v2/rerank",
"/claude_code_gateway/v1/messages",
"/claude_code_gateway/v1/messages/count_tokens",
"/cursor/chat/completions",
)
)
_MANAGED_MODEL_PATHS: Final = (
"/engines/{model:path}/chat/completions",
"/engines/{model:path}/completions",
"/engines/{model:path}/embeddings",
"/openai/deployments/{model:path}/chat/completions",
"/openai/deployments/{model:path}/completions",
"/openai/deployments/{model:path}/embeddings",
"/openai/deployments/{model:path}/images/generations",
"/openai/deployments/{model:path}/images/edits",
"/v1beta/models/{model_name:path}:countTokens",
"/v1beta/models/{model_name:path}:generateContent",
"/v1beta/models/{model_name:path}:streamGenerateContent",
"/models/{model_name:path}:countTokens",
"/models/{model_name:path}:generateContent",
"/models/{model_name:path}:streamGenerateContent",
)
_MANAGED_MCP_ROUTES: Final = tuple(
route for route in LiteLLMRoutes.mcp_inference_routes.value if route not in ("/token", "/introspect")
)
_MODEL_ROUTE_KINDS: Final[
Mapping[str, Literal["image_generation", "image_edit", "moderation", "speech", "body", "path"]]
] = MappingProxyType(
@ -94,17 +31,7 @@ _MODEL_ROUTE_KINDS: Final[
def managed_agent_route_allowed(route: str, method: str | None) -> bool:
from litellm.proxy.auth.route_checks import RouteChecks
if route in ("/agents", "/v1/agents"):
return method in (None, "GET", "HEAD")
if route in _MANAGED_REALTIME_ROUTES or route in _MANAGED_LIVE_ROUTES:
return method in (None, "GET")
if route in _MANAGED_MODEL_ROUTES or RouteChecks.check_route_access(route, _MANAGED_MODEL_PATHS):
return method in (None, "POST")
return RouteChecks.check_route_access(route, _MANAGED_MCP_ROUTES) or RouteChecks.check_route_access(
route, LiteLLMRoutes.agent_inference_routes.value
)
return auth_managed_agent_route_allowed(route, method)
def managed_inference_request(
@ -115,11 +42,8 @@ def managed_inference_request(
path_model: object = None,
query_model: object = None,
) -> dict[str, object]:
from litellm.proxy.auth.route_checks import RouteChecks
if route in _MANAGED_LIVE_ROUTES:
return dict(body)
if route in _MANAGED_REALTIME_ROUTES:
managed_route: Final = auth_managed_route(route)
if managed_route == "realtime":
from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model
model: Final = resolve_realtime_route_model(
@ -130,7 +54,7 @@ def managed_inference_request(
AgentIdentityFailure(message="Managed inference requires an explicit or configured model")
)
return {**body, "model": model}
if route not in _MANAGED_MODEL_ROUTES and not RouteChecks.check_route_access(route, _MANAGED_MODEL_PATHS):
if managed_route != "model":
return dict(body)
from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model

View file

@ -1,4 +1,3 @@
import itertools
import re
from collections.abc import Collection, Iterable, Mapping
from typing import Final, cast
@ -18,6 +17,14 @@ from litellm.proxy._types import (
ProxyException,
UserAPIKeyAuth,
)
from litellm.rust_bridge._native import (
auth_allowed_routes_grant,
auth_first_denied_route,
auth_jwt_team_routes_grant,
auth_route_matches_denied,
auth_route_matches_prefix,
auth_route_matches_wildcard,
)
from .auth_checks_organization import ( # noqa: F401 # legacy module exports
_user_is_org_admin, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
@ -111,6 +118,10 @@ _TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "co
_TOKEN_COUNTING_ACTION: Final = "countTokens"
def _allowed_passthrough_routes(metadata: Mapping[str, object] | None) -> object:
return (metadata or {}).get("allowed_passthrough_routes")
class RouteChecks:
@staticmethod
def check_route_enabled(route: str) -> None:
@ -659,58 +670,13 @@ class RouteChecks:
@staticmethod
def route_matches_wildcard_pattern(route: str, pattern: str) -> bool:
"""
Check if route matches the wildcard pattern
eg.
pattern: "/scim/v2/*"
route: "/scim/v2/Users"
- returns: True
pattern: "/scim/v2/*"
route: "/chat/completions"
- returns: False
pattern: "/scim/v2/*"
route: "/scim/v2/Users/123"
- returns: True
"""
if pattern.endswith("*"):
# Get the prefix (everything before the wildcard)
prefix: Final = pattern[:-1]
return route.startswith(prefix)
else:
# If there's no wildcard, the pattern and route should match exactly
return route == pattern
"""A trailing `*` matches by prefix, anything else exactly."""
return auth_route_matches_wildcard(route, pattern)
@staticmethod
def _route_matches_allowed_route(route: str, allowed_route: str) -> bool:
"""
Check if route matches the allowed_route pattern.
Supports both exact match and prefix match.
Examples:
- allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-6" -> True (exact match)
- allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-6/v1/chat/completions" -> True (prefix match)
- allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-600" -> False (not a valid prefix)
Args:
route: The actual route being accessed
allowed_route: The allowed route pattern
Returns:
bool: True if route matches (exact or prefix), False otherwise
"""
# Exact match
if route == allowed_route:
return True
# Prefix match - ensure we add "/" to prevent false matches like /fake-openai-proxy-600
if route.startswith(allowed_route + "/"):
return True
return False
"""The route itself or anything under it, never a sibling that only shares the prefix."""
return auth_route_matches_prefix(route, allowed_route)
@staticmethod
def check_route_access(route: str, allowed_routes: Collection[str]) -> bool:
@ -813,13 +779,16 @@ class RouteChecks:
@staticmethod
def _route_matches_denied_route(route: str, denied_route: str) -> bool:
"""A `/` entry denies every route, since every route sits under the root."""
normalized_denied_route: Final = denied_route.rstrip("/") or "/"
return (
normalized_denied_route == "/"
or RouteChecks._route_matches_allowed_route(route=route, allowed_route=normalized_denied_route)
or RouteChecks.route_matches_wildcard_pattern(route=route, pattern=denied_route)
return auth_route_matches_denied(route, denied_route)
@staticmethod
def forwarded_pass_through_routes(route: str) -> tuple[str, ...]:
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass-through endpoints import this module
InitPassThroughEndpointHelpers,
)
return InitPassThroughEndpointHelpers.forwarded_routes(route)
@staticmethod
def matching_denied_passthrough_route(
route: str, metadata_sources: Iterable[Mapping[str, object] | None]
@ -828,32 +797,10 @@ class RouteChecks:
First ``denied_passthrough_routes`` entry across ``metadata_sources`` that matches ``route``.
Unlike the allowlist (key list, else team list), every source's deny list applies.
"""
denied_routes: Final = tuple(
itertools.chain.from_iterable(
cast( # cast-ok: management endpoints validate this metadata key as a list of route strings on write
"list[str]", (metadata or {}).get("denied_passthrough_routes") or []
)
for metadata in metadata_sources
)
)
if not denied_routes:
denied_sources: Final = [(metadata or {}).get("denied_passthrough_routes") for metadata in metadata_sources]
if not any(denied_sources):
return None
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
InitPassThroughEndpointHelpers,
)
forwarded_routes: Final = InitPassThroughEndpointHelpers.forwarded_routes(route)
return next(
(
denied_route
for denied_route in denied_routes
if any(
RouteChecks._route_matches_denied_route(route=candidate, denied_route=denied_route)
for candidate in forwarded_routes
)
),
None,
)
return auth_first_denied_route(list(RouteChecks.forwarded_pass_through_routes(route)), denied_sources)
@staticmethod
def passthrough_route_denied_exception(route: str, denied_route: str) -> HTTPException:
@ -881,11 +828,7 @@ class RouteChecks:
a named route group like ``openai_routes`` is only ever compared as a path, and an entry that names
no path segment (``*``, ``/*``) is skipped.
"""
return any(
RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route)
for allowed_route in team_allowed_routes
if allowed_route.rstrip("*").strip("/")
)
return auth_jwt_team_routes_grant(route, team_allowed_routes)
@staticmethod
def _jwt_team_allowed_routes(valid_token: UserAPIKeyAuth) -> Collection[str]:
@ -917,33 +860,13 @@ class RouteChecks:
@staticmethod
def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:
"""
Check if route is a passthrough route.
Supports both exact match and prefix match.
"""
metadata: Final = user_api_key_dict.metadata
team_metadata: Final = user_api_key_dict.team_metadata or {}
if metadata is None and team_metadata is None:
return False
if "allowed_passthrough_routes" not in metadata and "allowed_passthrough_routes" not in team_metadata:
return False
if (
metadata.get("allowed_passthrough_routes") is None
and team_metadata.get("allowed_passthrough_routes") is None
):
return False
allowed_passthrough_routes: Final = (
metadata.get("allowed_passthrough_routes") or team_metadata.get("allowed_passthrough_routes") or []
"""The key's ``allowed_passthrough_routes`` when it lists any, otherwise the team's, matched by prefix."""
return auth_allowed_routes_grant(
route,
key=_allowed_passthrough_routes(user_api_key_dict.metadata),
team=_allowed_passthrough_routes(user_api_key_dict.team_metadata),
)
# Check if route matches any allowed passthrough route (exact or prefix match)
for allowed_route in allowed_passthrough_routes:
if RouteChecks._route_matches_allowed_route(route=route, allowed_route=allowed_route):
return True
return False
@staticmethod
def is_assistants_api_request(request: Request) -> bool:
"""

View file

@ -3854,19 +3854,33 @@ dependencies = [
name = "litellm-config"
version = "0.1.0"
dependencies = [
"litellm-error",
"litellm-python-compat",
"litellm-router-types",
"litellm-secret-value",
"litellm-starlette",
"rstest",
"serde",
"serde_json",
"serde_with",
"serde_yaml_ng",
"strum",
"tempfile",
"thiserror 2.0.19",
"veil",
]
[[package]]
name = "litellm-config-load"
version = "0.1.0"
dependencies = [
"litellm-config",
"litellm-error",
"rstest",
"serde",
"serde_yaml_ng",
"tempfile",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-core-utils"
version = "0.1.0"
@ -3979,6 +3993,7 @@ dependencies = [
"http-body-util",
"litellm-circuit-breaker",
"litellm-config",
"litellm-config-load",
"litellm-core-utils",
"litellm-error",
"litellm-gateway-auth",
@ -4036,6 +4051,18 @@ dependencies = [
"veil",
]
[[package]]
name = "litellm-gateway-auth-core"
version = "0.1.0"
dependencies = [
"http 1.4.2",
"litellm-config",
"litellm-starlette",
"rstest",
"strum",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-gateway-forward"
version = "0.1.0"
@ -4565,6 +4592,7 @@ dependencies = [
"criterion",
"fancy-regex 0.19.2",
"futures-util",
"http 1.4.2",
"indexmap 2.14.0",
"litellm-auth",
"litellm-auth-aws",
@ -4577,9 +4605,11 @@ dependencies = [
"litellm-callbacks-python",
"litellm-callbacks-s3-log",
"litellm-circuit-breaker",
"litellm-config",
"litellm-core-utils",
"litellm-db",
"litellm-error",
"litellm-gateway-auth-core",
"litellm-host",
"litellm-host-python",
"litellm-http",
@ -4609,6 +4639,7 @@ dependencies = [
"litellm-storage-clickhouse",
"litellm-token-counter",
"litellm-tracing",
"parking_lot",
"prost",
"pyo3",
"pyo3-async-runtimes",
@ -4641,6 +4672,7 @@ dependencies = [
"serde",
"serde-pickle",
"serde_json",
"serde_with",
"thiserror 2.0.19",
]
@ -5026,6 +5058,15 @@ dependencies = [
"wiremock",
]
[[package]]
name = "litellm-starlette"
version = "0.1.0"
dependencies = [
"rstest",
"serde",
"strum",
]
[[package]]
name = "litellm-storage-clickhouse"
version = "0.1.0"

View file

@ -10,6 +10,7 @@ repository = "https://github.com/BerriAI/litellm"
[workspace.dependencies]
litellm-config = { path = "crates/config" }
litellm-config-load = { path = "crates/config-load" }
litellm-error = { path = "crates/error" }
litellm-router = { path = "crates/router" }
litellm-router-types = { path = "crates/router-types" }
@ -29,6 +30,7 @@ litellm-gateway = { path = "crates/gateway" }
litellm-gateway-forward = { path = "crates/gateway-forward" }
litellm-gateway-inference = { path = "crates/gateway-inference" }
litellm-gateway-auth = { path = "crates/gateway-auth" }
litellm-gateway-auth-core = { path = "crates/gateway-auth-core" }
litellm-gateway-management = { path = "crates/gateway-management" }
litellm-gateway-ui = { path = "crates/gateway-ui" }
litellm-coroutine = { path = "crates/coroutine" }
@ -42,6 +44,7 @@ litellm-callbacks-budgets = { path = "crates/callbacks-budgets" }
litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" }
litellm-framer = { path = "crates/framer" }
litellm-fnmatch = { path = "crates/fnmatch" }
litellm-starlette = { path = "crates/starlette" }
litellm-auth = { path = "crates/auth" }
litellm-auth-types = { path = "crates/auth-types" }
litellm-secret-value = { path = "crates/secret-value" }

View file

@ -0,0 +1,4 @@
# Config load
- This crate reads config files: the root file and every file its `include` names, merged breadth first, each file once. The typed `Config` lives in `litellm-config`, which reads no file and gets the merged document through `Config::from_value`
- A data crate never depends on this one. Only a host that starts from a path on disk (the gateway) does

View file

@ -0,0 +1,18 @@
[package]
name = "litellm-config-load"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "Reads a LiteLLM config file and the files its `include` names into one typed Config"
[dependencies]
litellm-config.workspace = true
serde.workspace = true
serde_yaml_ng = "0.10.0"
thiserror.workspace = true
[dev-dependencies]
litellm-error.workspace = true
rstest.workspace = true
tempfile.workspace = true

View file

@ -0,0 +1,14 @@
use std::{io, path::PathBuf};
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("could not read config {}", path.display())]
Read { path: PathBuf, source: io::Error },
#[error("invalid YAML config {}", path.display())]
ParseFile {
path: PathBuf,
source: serde_yaml_ng::Error,
},
#[error(transparent)]
Config(#[from] litellm_config::Error),
}

View file

@ -0,0 +1,13 @@
mod error;
mod includes;
use std::path::Path;
use litellm_config::Config;
pub use error::Error;
/// The config at `path` with every file its `include` names merged in.
pub fn load(path: impl AsRef<Path>) -> Result<Config, Error> {
Ok(Config::from_value(includes::load(path.as_ref())?)?)
}

View file

@ -0,0 +1,106 @@
use litellm_config_load::Error;
use rstest::{fixture, rstest};
use tempfile::TempDir;
#[fixture]
fn directory() -> TempDir {
tempfile::tempdir().unwrap()
}
#[fixture]
fn model_list_yaml() -> &'static str {
r#"
model_list:
- model_name: assistant
litellm_params:
model: anthropic/test-model
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: local
litellm_params:
model: test-model
api_base: http://localhost:8000/v1
custom_llm_provider: openai
"#
}
#[rstest]
fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) {
let path = directory.path().join("config.yaml");
std::fs::write(&path, model_list_yaml).unwrap();
let config = litellm_config_load::load(path).unwrap();
assert_eq!(config.model_list.len(), 2);
let anthropic = &config.model_list[0];
assert_eq!(anthropic.model_name, "assistant");
assert_eq!(anthropic.litellm_params.model, "anthropic/test-model");
assert_eq!(
anthropic.litellm_params.api_key.as_ref().unwrap().expose(),
"os.environ/ANTHROPIC_API_KEY"
);
assert!(anthropic.litellm_params.api_base.is_none());
assert!(anthropic.litellm_params.custom_llm_provider.is_none());
let local = &config.model_list[1];
assert_eq!(local.model_name, "local");
assert_eq!(local.litellm_params.model, "test-model");
assert!(local.litellm_params.api_key.is_none());
assert_eq!(
local.litellm_params.api_base.as_deref(),
Some("http://localhost:8000/v1")
);
assert_eq!(
local.litellm_params.custom_llm_provider.as_deref(),
Some("openai")
);
}
#[rstest]
fn distinguishes_read_errors_from_parse_errors(directory: TempDir) {
let error = litellm_config_load::load(directory.path().join("missing.yaml")).unwrap_err();
assert!(
litellm_error::Report::new(&error)
.to_string()
.contains("missing.yaml")
);
assert!(matches!(
error,
Error::Read { source, .. } if source.kind() == std::io::ErrorKind::NotFound
));
}
#[rstest]
#[case::policy_pipeline("../../../proxy/example_config_yaml/test_pipeline_config.yaml")]
#[case::gateway("../../../../tests/e2e/gateway/stage_mirror_ci_config.yml")]
#[case::pass_through("../../../proxy/example_config_yaml/pass_through_config.yaml")]
fn parses_representative_python_configs(#[case] relative_path: &str) {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../config")
.join(relative_path);
let config = litellm_config_load::load(path).unwrap();
assert!(!config.model_list.is_empty());
}
#[rstest]
fn resolves_nested_includes_once_in_breadth_first_order() {
let directory = TempDir::new().unwrap();
let root = directory.path();
std::fs::create_dir(root.join("nested")).unwrap();
std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap();
std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap();
std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap();
std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap();
let config = litellm_config_load::load(root.join("config.yaml")).unwrap();
assert_eq!(
config
.model_list
.iter()
.map(|model| model.model_name.as_str())
.collect::<Vec<_>>(),
["root", "first", "second", "third"]
);
assert_eq!(
config.general_settings.master_key.unwrap().expose(),
"third"
);
assert!(config.include.is_empty());
}

View file

@ -1,3 +1,11 @@
## Scope
- This crate is the typed data model of a config and reads no file. Reading files and `include` live in `config-load`, which hands the merged document to `Config::from_value`
## Redis settings
- This crate only deserializes YAML. `RedisSettings` is the raw `coordination_redis` or `cache_params` map and `CacheSettings` tells a `type: redis` cache apart from the rest. Reading those maps the way `litellm/_redis.py` does belongs to `gateway/src/redis`, and turning merged kwargs into a `Resolution` belongs to `redis-resolve`
## Route settings
- `RouteList` is the typed form of a route list in config and key metadata, and both hosts read through it. A value that is not a list of strings is `RouteList::Malformed`, which grants nothing. A route template inside such a list is starlette syntax, so it is parsed by `litellm-starlette`, never here

View file

@ -6,16 +6,17 @@ license.workspace = true
repository.workspace = true
[dependencies]
litellm-python-compat.workspace = true
litellm-router-types.workspace = true
litellm-secret-value.workspace = true
litellm-starlette.workspace = true
serde.workspace = true
serde_with.workspace = true
serde_yaml_ng = "0.10.0"
strum.workspace = true
thiserror.workspace = true
veil.workspace = true
[dev-dependencies]
litellm-error.workspace = true
rstest.workspace = true
serde_json.workspace = true
tempfile.workspace = true

View file

@ -1,14 +1,5 @@
use std::{io, path::PathBuf};
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("could not read config {}", path.display())]
Read { path: PathBuf, source: io::Error },
#[error("invalid YAML config")]
Parse(#[from] serde_yaml_ng::Error),
#[error("invalid YAML config {}", path.display())]
ParseFile {
path: PathBuf,
source: serde_yaml_ng::Error,
},
}

View file

@ -1,27 +1,28 @@
mod error;
mod includes;
mod mcp;
mod model;
mod pass_through;
mod redis;
mod routes;
mod settings;
mod value;
use std::{fmt, path::Path};
use serde::Deserialize;
pub use error::Error;
pub use litellm_router_types::{LitellmParams, Spelled};
pub use mcp::{McpAuth, McpServer, McpTransport};
pub use model::Model;
pub use pass_through::{PassThroughEndpoint, PassThroughGuardrail};
pub use redis::{CacheSettings, RedisSettings};
pub use routes::RouteList;
pub use settings::{
ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings,
TracingStoreSettings,
};
pub use value::{AdditionalFields, Object, Value};
#[derive(Clone, Default, Deserialize)]
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(default)]
pub struct Config {
pub model_list: Box<[Model]>,
@ -52,45 +53,12 @@ pub struct Config {
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for Config {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Config")
.field("model_list", &self.model_list)
.field("general_settings", &self.general_settings)
.field("router_settings", &self.router_settings)
.field("litellm_settings", &self.litellm_settings)
.field("environment_variables", &self.environment_variables)
.field("callback_settings", &self.callback_settings)
.field("assistant_settings", &self.assistant_settings)
.field("default_vertex_config", &self.default_vertex_config)
.field("mcp_servers", &self.mcp_servers)
.field("credential_list", &self.credential_list)
.field("guardrails", &self.guardrails)
.field("prompts", &self.prompts)
.field("sandbox_tools", &self.sandbox_tools)
.field("search_tools", &self.search_tools)
.field("files_settings", &self.files_settings)
.field("finetune_settings", &self.finetune_settings)
.field("mcp_tools", &self.mcp_tools)
.field("vector_store_registry", &self.vector_store_registry)
.field("worker_registry", &self.worker_registry)
.field("agents", &self.agents)
.field("agent_list", &self.agent_list)
.field("policies", &self.policies)
.field("policy_attachments", &self.policy_attachments)
.field("include", &self.include)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
}
impl Config {
pub fn from_yaml(yaml: &str) -> Result<Self, Error> {
Ok(serde_yaml_ng::from_str(yaml)?)
}
pub fn load(path: impl AsRef<Path>) -> Result<Self, Error> {
Ok(serde_yaml_ng::from_value(includes::load(path.as_ref())?)?)
pub fn from_value(document: Value) -> Result<Self, Error> {
Ok(serde_yaml_ng::from_value(document)?)
}
}

View file

@ -1,11 +1,13 @@
use std::{collections::BTreeMap, fmt};
use std::collections::BTreeMap;
use litellm_secret_value::SecretValue;
use serde::Deserialize;
use veil::Redact;
use crate::Object;
#[derive(Clone, Default, Deserialize)]
/// `command` and `args` can carry credentials, and the `SecretValue`s redact themselves.
#[derive(Clone, Default, Deserialize, Redact)]
#[serde(default)]
pub struct McpServer {
pub server_id: Option<String>,
@ -14,7 +16,9 @@ pub struct McpServer {
pub mcp_info: Object,
pub transport: McpTransport,
pub url: Option<SecretValue>,
#[redact(fixed = 3)]
pub command: Option<String>,
#[redact(fixed = 3)]
pub args: Box<[String]>,
pub env: BTreeMap<String, SecretValue>,
pub auth_type: Option<McpAuth>,
@ -29,18 +33,6 @@ pub struct McpServer {
pub unsupported: Object,
}
impl fmt::Debug for McpServer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("McpServer")
.field("transport", &self.transport)
.field("auth_type", &self.auth_type)
.field("timeout", &self.timeout)
.field("max_concurrent_requests", &self.max_concurrent_requests)
.finish_non_exhaustive()
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
#[serde(rename_all = "snake_case")]
pub enum McpTransport {

View file

@ -1,11 +1,9 @@
use std::fmt;
use litellm_router_types::LitellmParams;
use serde::Deserialize;
use crate::{AdditionalFields, Object};
#[derive(Clone, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
pub struct Model {
pub model_name: String,
pub litellm_params: LitellmParams,
@ -15,16 +13,3 @@ pub struct Model {
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for Model {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Model")
.field("model_name", &self.model_name)
.field("litellm_params", &self.litellm_params)
.field("model_info", &self.model_info)
.field("blocked", &self.blocked)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
}

View file

@ -0,0 +1,61 @@
use std::collections::BTreeMap;
use litellm_python_compat::lenient::{IsTrue, Truthy};
use litellm_secret_value::SecretValue;
use litellm_starlette::RouteTemplate;
use serde::Deserialize;
use serde_with::serde_as;
use crate::{Object, Spelled};
/// A `general_settings.pass_through_endpoints` entry: a proxy `path` forwarded to `target`. Each
/// field is read the way `initialize_pass_through_endpoints` reads the config dict.
#[serde_as]
#[derive(Clone, Debug, Deserialize)]
pub struct PassThroughEndpoint {
pub id: Option<String>,
pub path: RouteTemplate,
/// Registration skips an endpoint without one.
pub target: Option<SecretValue>,
/// Upstream credentials and `os.environ/` references, so only the keys are printed.
#[serde(default)]
pub headers: Object,
#[serde(default)]
pub default_query_params: Object,
/// `endpoint.get("include_subpath", False) is True`
#[serde_as(as = "IsTrue")]
#[serde(default)]
pub include_subpath: bool,
pub cost_per_request: Option<Spelled<f64>>,
pub timeout: Option<Spelled<f64>>,
/// `endpoint.get("auth", True) is True`, which decides whether a request needs a key.
/// Registration still reads `str(auth).lower() == "true"`, until both share one rule.
#[serde_as(as = "IsTrue")]
#[serde(default = "auth_default")]
pub auth: bool,
pub methods: Option<Box<[String]>>,
pub guardrails: Option<BTreeMap<String, Option<PassThroughGuardrail>>>,
#[serde_as(as = "Truthy")]
#[serde(default)]
pub forward_headers: bool,
#[serde_as(as = "Truthy")]
#[serde(default)]
pub merge_query_params: bool,
/// `"langfuse"` reads the key out of a Basic credential.
pub custom_auth_parser: Option<String>,
#[serde(flatten)]
pub additional_fields: Object,
}
const fn auth_default() -> bool {
true
}
/// `PassThroughGuardrailSettings`: which request and response fields one guardrail reads.
#[derive(Clone, Debug, Deserialize)]
pub struct PassThroughGuardrail {
pub request_fields: Option<Box<[String]>>,
pub response_fields: Option<Box<[String]>>,
#[serde(flatten)]
pub additional_fields: Object,
}

View file

@ -0,0 +1,6 @@
use litellm_python_compat::lenient::Lenient;
/// A route list from config or key metadata, such as `public_routes` or
/// `allowed_passthrough_routes`. A value that is not a list of strings is `Malformed`, and each
/// reader decides what that grants: an allow list grants nothing, a deny list denies every route.
pub type RouteList = Lenient<Box<[String]>>;

View file

@ -1,10 +1,10 @@
use std::fmt;
use litellm_secret_value::SecretValue;
use serde::Deserialize;
use veil::Redact;
use crate::{
AdditionalFields, CacheSettings, Object, RedisSettings, Spelled, Value, value::one_or_many,
AdditionalFields, CacheSettings, Object, PassThroughEndpoint, RedisSettings, Spelled, Value,
value::one_or_many,
};
#[derive(Clone, Debug, Deserialize)]
@ -13,7 +13,7 @@ pub enum TracingStoreKind {
Clickhouse,
}
#[derive(Clone, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ClickHouseStoreSettings {
#[serde(rename = "type")]
@ -23,17 +23,6 @@ pub struct ClickHouseStoreSettings {
pub retention_days: Option<Spelled<f64>>,
}
impl fmt::Debug for ClickHouseStoreSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ClickHouseStoreSettings")
.field("kind", &self.kind)
.field("database", &self.database)
.field("retention_days", &self.retention_days)
.finish()
}
}
#[derive(Clone, Debug, Deserialize)]
#[serde(untagged)]
pub enum TracingStoreSettings {
@ -48,7 +37,7 @@ pub struct TracingSettings {
pub additional_fields: AdditionalFields,
}
#[derive(Clone, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
#[serde(default)]
pub struct GeneralSettings {
pub completion_model: Option<String>,
@ -81,6 +70,7 @@ pub struct GeneralSettings {
pub coordination_redis: Option<RedisSettings>,
pub mcp_allowed_hosts: Option<Box<[String]>>,
pub mcp_allowed_origins: Box<[String]>,
pub pass_through_endpoints: Option<Box<[PassThroughEndpoint]>>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
@ -118,38 +108,13 @@ impl Default for GeneralSettings {
coordination_redis: None,
mcp_allowed_hosts: None,
mcp_allowed_origins: Box::default(),
additional_fields: AdditionalFields::new(),
pass_through_endpoints: None,
additional_fields: AdditionalFields::default(),
}
}
}
impl fmt::Debug for GeneralSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("GeneralSettings")
.field("completion_model", &self.completion_model)
.field(
"max_in_flight_requests_per_worker",
&self.max_in_flight_requests_per_worker,
)
.field(
"max_queued_requests_per_worker",
&self.max_queued_requests_per_worker,
)
.field(
"admission_queue_timeout_seconds",
&self.admission_queue_timeout_seconds,
)
.field("master_key", &self.master_key)
.field("database_url", &self.database_url)
.field("tracing", &self.tracing)
.field("store_model_in_db", &self.store_model_in_db)
.field("additional_fields", &self.additional_fields.keys())
.finish_non_exhaustive()
}
}
#[derive(Clone, Default, Deserialize)]
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(default)]
pub struct RouterSettings {
pub routing_strategy: Option<String>,
@ -175,38 +140,8 @@ pub struct RouterSettings {
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for RouterSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RouterSettings")
.field("routing_strategy", &self.routing_strategy)
.field("routing_strategy_args", &self.routing_strategy_args)
.field("routing_groups", &self.routing_groups)
.field("retry_policy", &self.retry_policy)
.field("model_group_retry_policy", &self.model_group_retry_policy)
.field(
"model_group_affinity_config",
&self.model_group_affinity_config,
)
.field("allowed_fails", &self.allowed_fails)
.field("cooldown_time", &self.cooldown_time)
.field("num_retries", &self.num_retries)
.field("timeout", &self.timeout)
.field("max_retries", &self.max_retries)
.field("retry_after", &self.retry_after)
.field("fallbacks", &self.fallbacks)
.field("context_window_fallbacks", &self.context_window_fallbacks)
.field("model_group_alias", &self.model_group_alias)
.field("enable_tag_filtering", &self.enable_tag_filtering)
.field("weights", &self.weights)
.field("tag_routing_prefix", &self.tag_routing_prefix)
.field("optional_pre_call_checks", &self.optional_pre_call_checks)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
}
#[derive(Clone, Default, Deserialize)]
/// Callback entries can carry credentials inline, so only whether they are set is printed.
#[derive(Clone, Default, Deserialize, Redact)]
#[serde(default)]
pub struct LiteLlmSettings {
pub ssl_verify: Option<Spelled<bool>>,
@ -224,47 +159,16 @@ pub struct LiteLlmSettings {
pub cache: Option<bool>,
pub cache_params: Option<CacheSettings>,
#[serde(deserialize_with = "one_or_many")]
#[redact(fixed = 3)]
pub callbacks: Option<Vec<Value>>,
#[serde(deserialize_with = "one_or_many")]
#[redact(fixed = 3)]
pub success_callback: Option<Vec<Value>>,
#[serde(deserialize_with = "one_or_many")]
#[redact(fixed = 3)]
pub failure_callback: Option<Vec<Value>>,
pub json_logs: Option<bool>,
pub set_verbose: Option<bool>,
#[serde(flatten)]
pub additional_fields: AdditionalFields,
}
impl fmt::Debug for LiteLlmSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LiteLlmSettings")
.field("drop_params", &self.drop_params)
.field("request_timeout", &self.request_timeout)
.field("num_retries", &self.num_retries)
.field("cache", &self.cache)
.field("cache_params", &self.cache_params)
.field(
"callbacks",
&self.callbacks.as_ref().map(|callbacks| callbacks.len()),
)
.field(
"success_callback",
&self
.success_callback
.as_ref()
.map(|callbacks| callbacks.len()),
)
.field(
"failure_callback",
&self
.failure_callback
.as_ref()
.map(|callbacks| callbacks.len()),
)
.field("json_logs", &self.json_logs)
.field("set_verbose", &self.set_verbose)
.field("additional_fields", &self.additional_fields.keys())
.finish()
}
}

View file

@ -3,7 +3,8 @@ use std::{collections::BTreeMap, fmt, ops::Deref};
use serde::{Deserialize, Deserializer};
pub type Value = serde_yaml_ng::Value;
pub type AdditionalFields = BTreeMap<String, Value>;
/// The keys a config section has beyond its typed fields, printed as keys only.
pub type AdditionalFields = Object;
#[derive(Clone, Default, Deserialize)]
#[serde(transparent)]

View file

@ -1,57 +1,5 @@
use litellm_config::{Config, Error, Spelled, TracingStoreSettings};
use rstest::{fixture, rstest};
use tempfile::TempDir;
#[fixture]
fn directory() -> TempDir {
tempfile::tempdir().unwrap()
}
#[fixture]
fn model_list_yaml() -> &'static str {
r#"
model_list:
- model_name: assistant
litellm_params:
model: anthropic/test-model
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: local
litellm_params:
model: test-model
api_base: http://localhost:8000/v1
custom_llm_provider: openai
"#
}
#[rstest]
fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) {
let path = directory.path().join("config.yaml");
std::fs::write(&path, model_list_yaml).unwrap();
let config = Config::load(path).unwrap();
assert_eq!(config.model_list.len(), 2);
let anthropic = &config.model_list[0];
assert_eq!(anthropic.model_name, "assistant");
assert_eq!(anthropic.litellm_params.model, "anthropic/test-model");
assert_eq!(
anthropic.litellm_params.api_key.as_ref().unwrap().expose(),
"os.environ/ANTHROPIC_API_KEY"
);
assert!(anthropic.litellm_params.api_base.is_none());
assert!(anthropic.litellm_params.custom_llm_provider.is_none());
let local = &config.model_list[1];
assert_eq!(local.model_name, "local");
assert_eq!(local.litellm_params.model, "test-model");
assert!(local.litellm_params.api_key.is_none());
assert_eq!(
local.litellm_params.api_base.as_deref(),
Some("http://localhost:8000/v1")
);
assert_eq!(
local.litellm_params.custom_llm_provider.as_deref(),
Some("openai")
);
}
use rstest::rstest;
#[rstest]
fn config_debug_redacts_api_keys() {
@ -82,20 +30,6 @@ fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) {
assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_))));
}
#[rstest]
fn distinguishes_read_errors_from_parse_errors(directory: TempDir) {
let error = Config::load(directory.path().join("missing.yaml")).unwrap_err();
assert!(
litellm_error::Report::new(&error)
.to_string()
.contains("missing.yaml")
);
assert!(matches!(
error,
Error::Read { source, .. } if source.kind() == std::io::ErrorKind::NotFound
));
}
#[rstest]
#[case::literal("secret-master-key")]
#[case::reference("os.environ/LITELLM_MASTER_KEY")]
@ -334,15 +268,6 @@ include:
assert_eq!(&*config.include, &["models.yaml"]);
}
#[rstest]
#[case::policy_pipeline("../../../proxy/example_config_yaml/test_pipeline_config.yaml")]
#[case::gateway("../../../../tests/e2e/gateway/stage_mirror_ci_config.yml")]
fn parses_representative_python_configs(#[case] relative_path: &str) {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative_path);
let config = Config::load(path).unwrap();
assert!(!config.model_list.is_empty());
}
#[rstest]
#[case::one("callbacks: custom_callbacks.logger", 1)]
#[case::many("callbacks: [prometheus, otel]", 2)]
@ -387,32 +312,6 @@ fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secre
assert!(!format!("{config:?}").contains(secret));
}
#[rstest]
fn resolves_nested_includes_once_in_breadth_first_order() {
let directory = TempDir::new().unwrap();
let root = directory.path();
std::fs::create_dir(root.join("nested")).unwrap();
std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap();
std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap();
std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap();
std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap();
let config = Config::load(root.join("config.yaml")).unwrap();
assert_eq!(
config
.model_list
.iter()
.map(|model| model.model_name.as_str())
.collect::<Vec<_>>(),
["root", "first", "second", "third"]
);
assert_eq!(
config.general_settings.master_key.unwrap().expose(),
"third"
);
assert!(config.include.is_empty());
}
#[rstest]
fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() {
let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap();
@ -466,3 +365,115 @@ fn non_redis_cache_settings_remain_opaque() {
Some(litellm_config::CacheSettings::Other(_))
));
}
fn pass_through(entry: &str) -> litellm_config::PassThroughEndpoint {
let config = Config::from_yaml(&format!(
"general_settings: {{pass_through_endpoints: [{entry}]}}"
))
.unwrap();
config.general_settings.pass_through_endpoints.unwrap()[0].clone()
}
#[rstest]
fn a_pass_through_endpoint_reads_its_python_fields() {
let endpoint = pass_through(
"{path: '/bria/{item_id:int}', target: https://upstream.example/v1, include_subpath: true, \
headers: {Authorization: os.environ/BRIA_KEY}, cost_per_request: '0.5', methods: [POST], \
guardrails: {pii: {request_fields: [text]}, toxicity: null}, custom_auth_parser: langfuse, display_name: Bria}",
);
assert_eq!(
endpoint.path,
litellm_starlette::RouteTemplate::parse("/bria/{item_id:int}")
);
assert_eq!(
endpoint.target.unwrap().expose(),
"https://upstream.example/v1"
);
assert!(endpoint.include_subpath);
assert_eq!(
endpoint.headers.keys().collect::<Vec<_>>(),
["Authorization"]
);
assert_eq!(endpoint.methods.as_deref(), Some(&["POST".to_owned()][..]));
let guardrails = endpoint.guardrails.unwrap();
assert_eq!(
guardrails["pii"]
.as_ref()
.unwrap()
.request_fields
.as_deref(),
Some(&["text".to_owned()][..])
);
assert!(guardrails["toxicity"].is_none());
assert_eq!(endpoint.custom_auth_parser.as_deref(), Some("langfuse"));
assert!(endpoint.additional_fields.contains_key("display_name"));
}
/// `endpoint.get("auth", True) is True`, `endpoint.get("include_subpath", False) is True` and
/// `bool(forward_headers)`, as the proxy reads them.
#[rstest]
#[case::nothing_set("", true, false, false)]
#[case::booleans(
"auth: false, include_subpath: true, forward_headers: true",
false,
true,
true
)]
#[case::strings(
"auth: 'true', include_subpath: 'true', forward_headers: 'false'",
false,
false,
true
)]
#[case::numbers("auth: 1, include_subpath: 1, forward_headers: 0", false, false, false)]
fn pass_through_flags_read_like_python(
#[case] flags: &str,
#[case] auth: bool,
#[case] include_subpath: bool,
#[case] forward_headers: bool,
) {
let separator = if flags.is_empty() { "" } else { ", " };
let endpoint = pass_through(&format!(
"{{path: /x, target: https://t.example{separator}{flags}}}"
));
assert_eq!(
(
endpoint.auth,
endpoint.include_subpath,
endpoint.forward_headers
),
(auth, include_subpath, forward_headers)
);
}
#[rstest]
fn a_pass_through_endpoint_without_a_path_fails_the_load_like_registration() {
assert!(
Config::from_yaml(
"general_settings: {pass_through_endpoints: [{target: https://t.example}]}"
)
.is_err()
);
}
#[rstest]
fn pass_through_debug_hides_the_target_and_header_values() {
let config = Config::from_yaml(
"general_settings: {pass_through_endpoints: [{path: /x, target: https://user:target-secret@t.example, headers: {Authorization: header-secret}}]}",
)
.unwrap();
let debug = format!("{config:?}");
assert!(!debug.contains("target-secret") && !debug.contains("header-secret"));
}
#[rstest]
#[case::callbacks("callbacks")]
#[case::success_callback("success_callback")]
#[case::failure_callback("failure_callback")]
fn debug_hides_inline_callback_settings(#[case] field: &str) {
let config = Config::from_yaml(&format!(
"litellm_settings: {{{field}: [{{token: callback-list-secret}}]}}"
))
.unwrap();
assert!(!format!("{config:?}").contains("callback-list-secret"));
}

View file

@ -0,0 +1,17 @@
use litellm_config::RouteList;
use rstest::rstest;
#[rstest]
#[case::list("[/a, /b/*]", RouteList::Valid(["/a".to_owned(), "/b/*".to_owned()].into()))]
#[case::empty("[]", RouteList::Valid([].into()))]
#[case::bare_string("/a/*", RouteList::Malformed)]
#[case::number("5", RouteList::Malformed)]
#[case::mapping("{a: 1}", RouteList::Malformed)]
#[case::non_string_entry("[/a, 5, /b/*]", RouteList::Malformed)]
#[case::null_entry("[/a, null]", RouteList::Malformed)]
fn route_lists_read(#[case] yaml: &str, #[case] expected: RouteList) {
assert_eq!(
serde_yaml_ng::from_str::<RouteList>(yaml).unwrap(),
expected
);
}

View file

@ -0,0 +1,16 @@
[package]
name = "litellm-gateway-auth-core"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
http.workspace = true
litellm-config.workspace = true
litellm-starlette.workspace = true
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true

View file

@ -0,0 +1,7 @@
pub mod routes;
#[cfg(test)]
mod test_support;
pub use routes::{
ManagedRoute, MissingRouteGroup, RouteAllowlist, RouteGroup, RouteGroups, matches_wildcard,
};

View file

@ -0,0 +1,10 @@
# Routes
Whether a route may be called
## `error.rs`
- Every error a file here can produce lives in `routes/error.rs`. No other file declares its own error type
- `MissingRouteGroup` is a settings build failure from `RouteGroups::new`, never a request denial, so it stays out of `crate::Error`
- Every message is in `#[error(...)]`, built from the variant's typed fields. Keep each opening sentence stable, since clients match on it
- A file that only answers a question (`passthrough.rs`, `managed.rs`, `table.rs`) returns a plain value and adds nothing here

View file

@ -0,0 +1,8 @@
use super::RouteGroup;
/// A settings build failure, never a request denial.
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
#[error("the host's route table has no {} group", <&str>::from(*group))]
pub struct MissingRouteGroup {
pub group: RouteGroup,
}

View file

@ -0,0 +1,224 @@
use std::{collections::HashMap, sync::Arc};
use strum::VariantArray;
use super::{MissingRouteGroup, table::RouteGroup, template::Template};
#[derive(Debug)]
pub(super) struct Entries {
pub(super) routes: Box<[String]>,
pub(super) templates: Box<[Template]>,
}
impl Entries {
pub(super) fn matches(&self, route: &str) -> bool {
self.templates
.iter()
.any(|template| template.matches(route))
}
fn matches_compiled(&self, route: &str) -> bool {
self.templates
.iter()
.any(|template| template.matches_compiled(route))
}
}
/// `LiteLLMRoutes` by member name, which the host hands in with its settings.
#[derive(Clone, Debug, Default)]
pub struct RouteGroups(Arc<HashMap<String, Entries>>);
/// An operator's route list paired with the groups its entries may name, so the list is only ever
/// read through `_allowed_routes_check`.
#[derive(Clone, Copy, Debug)]
pub struct RouteAllowlist<'a> {
entries: &'a [String],
groups: &'a RouteGroups,
}
impl<'a> RouteAllowlist<'a> {
pub fn permits(&self, route: &str) -> bool {
self.groups.is_route_allowed(route, self.entries)
}
pub fn entries(&self) -> &'a [String] {
self.entries
}
/// `get_actual_routes`
pub fn expanded(&self) -> Vec<String> {
self.groups.actual_routes(self.entries)
}
/// `jwt_team_routes_grant_pass_through`: explicit paths and trailing-wildcard prefixes grant,
/// group names and blanket entries (`*`, `/*`) never do.
pub fn grants_pass_through(&self, route: &str) -> bool {
self.entries
.iter()
.filter(|allowed| !allowed.trim_end_matches('*').trim_matches('/').is_empty())
.any(|allowed| matches_wildcard(route, allowed))
}
}
impl RouteGroups {
/// Fails on a table without a group the core's rules read, so a group the host renames stops
/// the settings build instead of reading as empty.
pub fn new(
groups: impl IntoIterator<Item = (String, Vec<String>)>,
) -> Result<Self, MissingRouteGroup> {
let groups: HashMap<String, Entries> = groups
.into_iter()
.map(|(name, routes)| {
let templates = routes
.iter()
.map(|route| Template::compile(route))
.collect();
(
name,
Entries {
routes: routes.into(),
templates,
},
)
})
.collect();
let missing = RouteGroup::VARIANTS
.iter()
.copied()
.find(|group| !groups.contains_key(<&str>::from(*group)));
match missing {
Some(group) => Err(MissingRouteGroup { group }),
None => Ok(Self(Arc::new(groups))),
}
}
pub(super) fn entries(&self, group: RouteGroup) -> Option<&Entries> {
self.0.get(<&str>::from(group))
}
/// `LiteLLMRoutes.<group>.value`
pub fn routes(&self, group: RouteGroup) -> &[String] {
self.entries(group)
.map(|entries| &*entries.routes)
.unwrap_or_default()
}
/// `route in LiteLLMRoutes.<group>.value`: the route is one of the group's entries as written.
pub fn lists(&self, route: &str, group: RouteGroup) -> bool {
self.routes(group).iter().any(|entry| entry == route)
}
/// The route matches one of the group's entries read as starlette templates.
pub fn matches(&self, route: &str, group: RouteGroup) -> bool {
self.entries(group)
.is_some_and(|entries| entries.matches(route))
}
/// `_allowed_routes_check` over an operator's list: each entry is a `LiteLLMRoutes` group name
/// whose templates match as starlette `compile_path` compiles them,
/// an exact route, or a prefix ending in `*`.
pub fn is_route_allowed<S: AsRef<str>>(&self, route: &str, allowed_routes: &[S]) -> bool {
allowed_routes
.iter()
.any(|allowed| match self.0.get(allowed.as_ref()) {
Some(entries) => entries.matches_compiled(route),
None => matches_wildcard(route, allowed.as_ref()),
})
}
pub fn allowlist<'a>(&'a self, entries: &'a [String]) -> RouteAllowlist<'a> {
RouteAllowlist {
entries,
groups: self,
}
}
/// `get_actual_routes`: group names expanded to their routes, anything else kept as given.
pub fn actual_routes<S: AsRef<str>>(&self, allowed_routes: &[S]) -> Vec<String> {
allowed_routes
.iter()
.flat_map(|allowed| match self.0.get(allowed.as_ref()) {
Some(entries) => entries.routes.to_vec(),
None => vec![allowed.as_ref().to_owned()],
})
.collect()
}
}
/// `route_matches_wildcard_pattern`: a trailing `*` matches by prefix, anything else exactly.
pub fn matches_wildcard(route: &str, pattern: &str) -> bool {
match pattern.strip_suffix('*') {
Some(prefix) => route.starts_with(prefix),
None => route == pattern,
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
use crate::test_support::{route_groups, route_table};
#[rstest]
#[case::exact("/scim/v2", "/scim/v2", true)]
#[case::wildcard_child("/scim/v2/Users/1", "/scim/v2/*", true)]
#[case::wildcard_other("/chat/completions", "/scim/v2/*", false)]
#[case::no_wildcard_child("/scim/v2/Users", "/scim/v2", false)]
fn wildcard(#[case] route: &str, #[case] pattern: &str, #[case] expected: bool) {
assert_eq!(matches_wildcard(route, pattern), expected);
}
#[rstest]
#[case::group("info_routes", "/key/info", true)]
#[case::group_template("openai_routes", "/engines/gpt/chat/completions", true)]
#[case::exact_route("/v1/messages", "/v1/messages", true)]
#[case::wildcard_prefix("/scim/v2/*", "/scim/v2/Users/1", true)]
#[case::wildcard_other("/scim/v2/*", "/scim/v3", false)]
#[case::unknown_name_is_exact("not_a_group", "/not_a_group", false)]
#[case::path_swallows_a_colon_like_compile_path(
"google_routes",
"/v1beta/models/a:b:generateContent",
true
)]
#[case::path_may_be_empty_like_compile_path(
"google_routes",
"/v1beta/models/:generateContent",
true
)]
#[case::path_stops_at_a_newline_like_compile_path(
"google_routes",
"/v1beta/models/a\nb:generateContent",
false
)]
fn allowed_routes(#[case] allowed: &str, #[case] route: &str, #[case] expected: bool) {
assert_eq!(route_groups().is_route_allowed(route, &[allowed]), expected);
}
#[test]
fn a_table_without_a_group_the_core_reads_is_refused() {
let table = route_table()
.into_iter()
.filter(|(name, _)| name != "public_routes");
assert_eq!(
RouteGroups::new(table).unwrap_err(),
MissingRouteGroup {
group: RouteGroup::Public
}
);
}
#[test]
fn actual_routes_expand_groups_in_order() {
assert_eq!(
route_groups().actual_routes(&["/first", "master_key_only_routes", "/last"]),
[
"/first",
"/global/spend/reset",
"/memory-usage-in-mem-cache",
"/memory-usage-in-mem-cache-items",
"/last",
]
);
}
}

View file

@ -0,0 +1,128 @@
use http::Method;
use super::{
groups::RouteGroups,
table::{
MANAGED_DISCOVERY_ROUTES, MANAGED_LIVE_ROUTES, MANAGED_MCP_EXCLUDED,
MANAGED_MODEL_EXTRA_ROUTES, MANAGED_MODEL_OPERATIONS, MANAGED_MODEL_PATHS,
MANAGED_MODEL_PREFIXES, MANAGED_REALTIME_ROUTES, RouteGroup,
},
};
impl RouteGroups {
fn is_managed_tool_route(&self, route: &str) -> bool {
let mcp = self
.entries(RouteGroup::McpInference)
.into_iter()
.flat_map(|entries| {
entries
.routes
.iter()
.zip(&entries.templates)
.filter(|(entry, _)| !MANAGED_MCP_EXCLUDED.contains(&entry.as_str()))
});
{ mcp }.any(|(_, template)| template.matches(route))
|| self.matches(route, RouteGroup::AgentInference)
}
pub fn managed_route(&self, route: &str) -> ManagedRoute {
if MANAGED_DISCOVERY_ROUTES.contains(&route) {
return ManagedRoute::Discovery;
}
if MANAGED_REALTIME_ROUTES.contains(&route) {
return ManagedRoute::Realtime;
}
if MANAGED_LIVE_ROUTES.contains(&route) {
return ManagedRoute::Live;
}
if is_managed_model_route(route) {
return ManagedRoute::Model;
}
if self.is_managed_tool_route(route) {
return ManagedRoute::Tool;
}
ManagedRoute::Other
}
/// `managed_agent_route_allowed`: agent discovery is read-only, realtime and live sessions open
/// with a GET, inference is a POST, and MCP and agent-to-agent calls take any method.
pub fn managed_agent_route_allowed(&self, route: &str, method: Option<&Method>) -> bool {
match self.managed_route(route) {
ManagedRoute::Discovery => {
matches!(method, None | Some(&Method::GET | &Method::HEAD))
}
ManagedRoute::Realtime | ManagedRoute::Live => {
matches!(method, None | Some(&Method::GET))
}
ManagedRoute::Model => matches!(method, None | Some(&Method::POST)),
ManagedRoute::Tool => true,
ManagedRoute::Other => false,
}
}
}
fn is_managed_model_route(route: &str) -> bool {
MANAGED_MODEL_EXTRA_ROUTES.contains(&route)
|| MANAGED_MODEL_PREFIXES.iter().any(|prefix| {
route
.strip_prefix(prefix)
.and_then(|rest| rest.strip_prefix('/'))
.is_some_and(|operation| MANAGED_MODEL_OPERATIONS.contains(&operation))
})
|| MANAGED_MODEL_PATHS.matches(route)
}
/// The kind of route a managed agent identity calls, which decides the methods it may use and how
/// its request names a model.
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr)]
pub enum ManagedRoute {
#[strum(serialize = "discovery")]
Discovery,
#[strum(serialize = "realtime")]
Realtime,
#[strum(serialize = "live")]
Live,
#[strum(serialize = "model")]
Model,
/// MCP and agent-to-agent calls.
#[strum(serialize = "tool")]
Tool,
#[strum(serialize = "other")]
Other,
}
#[cfg(test)]
mod tests {
use http::Method;
use rstest::rstest;
use crate::test_support::route_groups;
#[rstest]
#[case::discovery_get("/agents", Some(Method::GET), true)]
#[case::discovery_post("/v1/agents", Some(Method::POST), false)]
#[case::realtime_get("/v1/realtime", Some(Method::GET), true)]
#[case::live_post("/live/sessions", Some(Method::POST), false)]
#[case::chat_post("/v1/chat/completions", Some(Method::POST), true)]
#[case::chat_get("/chat/completions", Some(Method::GET), false)]
#[case::deployment_path("/openai/deployments/gpt/x/chat/completions", Some(Method::POST), true)]
#[case::google_verb("/v1beta/models/gemini:generateContent", None, true)]
#[case::path_model_with_newline("/engines/a\nb/chat/completions", Some(Method::POST), true)]
#[case::empty_path_model("/engines//completions", Some(Method::POST), false)]
#[case::google_model_with_colon("/models/a:b:generateContent", Some(Method::POST), false)]
#[case::google_second_verb("/models/a:generateContent:countTokens", Some(Method::POST), false)]
#[case::mcp_any_method("/mcp/tools/call", Some(Method::DELETE), true)]
#[case::mcp_token_endpoint("/token", Some(Method::POST), false)]
#[case::a2a("/a2a/agent-1/message/send", Some(Method::POST), true)]
#[case::management("/key/generate", Some(Method::POST), false)]
fn managed_agents_reach_inference_tools_and_discovery_only(
#[case] route: &str,
#[case] method: Option<Method>,
#[case] allowed: bool,
) {
assert_eq!(
route_groups().managed_agent_route_allowed(route, method.as_ref()),
allowed
);
}
}

View file

@ -0,0 +1,12 @@
mod error;
mod groups;
mod managed;
pub mod passthrough;
pub(crate) mod table;
mod template;
pub use error::MissingRouteGroup;
pub use groups::{RouteAllowlist, RouteGroups, matches_wildcard};
pub use managed::ManagedRoute;
pub use table::RouteGroup;
pub(crate) use template::RouteSet;

View file

@ -0,0 +1,271 @@
//! Grants for pass-through endpoints registered with `auth: true`. `RouteChecks` calls these through
//! `_native`, so the core and the Python key path share one implementation.
use litellm_config::RouteList;
use super::{RouteAllowlist, matches_wildcard};
/// A `denied_passthrough_routes` entry that matched decides `Denied`; with none, `Denied` means
/// nothing granted the route.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum PassthroughAccess {
Allowed,
Denied { denied_route: Option<String> },
}
/// A team's `allowed_passthrough_routes` and `denied_passthrough_routes` metadata. `allowed` is
/// `None` when the key is missing or `None`.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct PassthroughGrants {
pub allowed: Option<RouteList>,
pub denied: Option<RouteList>,
}
/// `InitPassThroughEndpointHelpers.forwarded_routes(route)`: every route the request is forwarded as.
#[derive(Clone, Copy, Debug, Default)]
pub struct ForwardedRoutes<'a>(pub &'a [String]);
/// A key's and its team's `allowed_passthrough_routes`.
#[derive(Clone, Copy, Debug, Default)]
pub struct AllowedPassthrough<'a> {
pub key: Option<&'a RouteList>,
pub team: Option<&'a RouteList>,
}
impl AllowedPassthrough<'_> {
/// `check_passthrough_route_access`: the key's list when it lists any, otherwise the team's.
/// A malformed key list is chosen over the team's and grants nothing.
pub fn grants(&self, route: &str) -> bool {
self.key
.filter(|allowed| !matches!(allowed, RouteList::Valid(routes) if routes.is_empty()))
.or(self.team)
.map(RouteList::items)
.unwrap_or_default()
.iter()
.any(|allowed| matches_prefix(route, allowed))
}
}
/// What a malformed `denied_passthrough_routes` reports as its match. It denies every route, so a
/// deny list that does not read never lets a route through.
pub const MALFORMED_DENIED_ROUTES: &str = "(not a list of routes)";
/// `denied_passthrough_routes` from a key's and a team's `metadata`, in that order
#[derive(Clone, Copy, Debug, Default)]
pub struct DeniedPassthrough<'a>(pub &'a [RouteList]);
impl<'a> DeniedPassthrough<'a> {
/// `matching_denied_passthrough_route`: the first entry that matches any route the request is
/// forwarded as.
pub fn first_match(&self, forwarded: ForwardedRoutes<'_>) -> Option<&'a str> {
self.0.iter().find_map(|list| match list {
RouteList::Malformed => Some(MALFORMED_DENIED_ROUTES),
RouteList::Valid(routes) => routes.iter().map(String::as_str).find(|denied| {
forwarded
.0
.iter()
.any(|route| matches_denied(route, denied))
}),
})
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct PassthroughRequest<'a> {
pub route: &'a str,
/// `RouteChecks.is_auth_enforced_pass_through_route(route, method)`
pub auth_enforced: bool,
pub forwarded_routes: ForwardedRoutes<'a>,
/// `litellm_jwtauth.team_allowed_routes`
pub jwt_team_allowed_routes: Option<RouteAllowlist<'a>>,
}
impl PassthroughRequest<'_> {
/// `team_has_passthrough_route_access`, naming the deny entry that matched when it refuses.
pub fn team_access(&self, team: &PassthroughGrants) -> PassthroughAccess {
if !self.auth_enforced {
return PassthroughAccess::Allowed;
}
let denied_route =
DeniedPassthrough(team.denied.as_slice()).first_match(self.forwarded_routes);
let team_grants = AllowedPassthrough {
key: None,
team: team.allowed.as_ref(),
};
if denied_route.is_none()
&& (self
.jwt_team_allowed_routes
.is_some_and(|routes| routes.grants_pass_through(self.route))
|| team_grants.grants(self.route))
{
return PassthroughAccess::Allowed;
}
PassthroughAccess::Denied {
denied_route: denied_route.map(str::to_owned),
}
}
}
/// `_route_matches_allowed_route`: the route itself or anything under it, never a sibling that
/// only shares the prefix.
pub fn matches_prefix(route: &str, allowed_route: &str) -> bool {
route
.strip_prefix(allowed_route)
.is_some_and(|rest| rest.is_empty() || rest.starts_with('/'))
}
/// `_route_matches_denied_route`. A `/` entry denies every route.
pub fn matches_denied(route: &str, denied_route: &str) -> bool {
let normalized = denied_route.trim_end_matches('/');
normalized.is_empty()
|| matches_prefix(route, normalized)
|| matches_wildcard(route, denied_route)
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
use crate::test_support::route_groups;
fn owned(routes: &[&str]) -> Vec<String> {
routes.iter().map(|route| (*route).to_owned()).collect()
}
fn list(routes: &[&str]) -> RouteList {
RouteList::Valid(owned(routes).into())
}
#[rstest]
#[case::exact("/proxy-6", "/proxy-6", true)]
#[case::child("/proxy-6/v1/chat", "/proxy-6", true)]
#[case::sibling("/proxy-600", "/proxy-6", false)]
#[case::shorter("/proxy", "/proxy-6", false)]
fn prefix(#[case] route: &str, #[case] allowed: &str, #[case] expected: bool) {
assert_eq!(matches_prefix(route, allowed), expected);
}
#[rstest]
#[case::root_denies_all("/anything", "/", true)]
#[case::trailing_slash("/openai/x", "/openai/", true)]
#[case::wildcard("/openai/x", "/openai/*", true)]
#[case::sibling("/openai-x", "/openai", false)]
fn denied(#[case] route: &str, #[case] denied_route: &str, #[case] expected: bool) {
assert_eq!(matches_denied(route, denied_route), expected);
}
#[rstest]
#[case::first_entry_that_matches_any_forwarded_route(&["/a", "/v1/x"], &["/b", "/v1/*", "/a"], Some("/v1/*"))]
#[case::none(&["/a"], &["/b"], None)]
fn first_denied(
#[case] forwarded: &[&str],
#[case] denied: &[&str],
#[case] expected: Option<&str>,
) {
let (forwarded, denied) = (owned(forwarded), [list(denied)]);
assert_eq!(
DeniedPassthrough(&denied).first_match(ForwardedRoutes(&forwarded)),
expected
);
}
#[rstest]
#[case::malformed_list_denies_every_route(
[list(&["/b"]), RouteList::Malformed],
Some(MALFORMED_DENIED_ROUTES)
)]
#[case::an_earlier_match_names_its_entry([list(&["/a"]), RouteList::Malformed], Some("/a"))]
fn malformed_deny_lists_deny(#[case] denied: [RouteList; 2], #[case] expected: Option<&str>) {
let forwarded = owned(&["/a"]);
assert_eq!(
DeniedPassthrough(&denied).first_match(ForwardedRoutes(&forwarded)),
expected
);
}
#[rstest]
#[case::explicit(&["/openai/x"], true)]
#[case::prefix_wildcard(&["/openai/*"], true)]
#[case::blanket_star(&["*", "/*"], false)]
#[case::route_group_name(&["openai_routes"], false)]
fn jwt_team_routes(#[case] allowed: &[&str], #[case] expected: bool) {
let (groups, allowed) = (route_groups(), owned(allowed));
assert_eq!(
groups.allowlist(&allowed).grants_pass_through("/openai/x"),
expected
);
}
#[rstest]
#[case::key_list_wins(Some(list(&["/a"])), Some(list(&["/openai"])), false)]
#[case::empty_key_list_falls_to_team(Some(list(&[])), Some(list(&["/openai"])), true)]
#[case::team_only(None, Some(list(&["/openai"])), true)]
#[case::neither(None, None, false)]
#[case::malformed_key_list_grants_nothing(Some(RouteList::Malformed), Some(list(&["/openai"])), false)]
#[case::malformed_team_list_grants_nothing(None, Some(RouteList::Malformed), false)]
fn allowed_routes(
#[case] key: Option<RouteList>,
#[case] team: Option<RouteList>,
#[case] expected: bool,
) {
let allowed = AllowedPassthrough {
key: key.as_ref(),
team: team.as_ref(),
};
assert_eq!(allowed.grants("/openai/x"), expected);
}
fn request<'a>(
forwarded: &'a [String],
jwt_team_allowed_routes: RouteAllowlist<'a>,
) -> PassthroughRequest<'a> {
PassthroughRequest {
route: "/openai/x",
auth_enforced: true,
forwarded_routes: ForwardedRoutes(forwarded),
jwt_team_allowed_routes: Some(jwt_team_allowed_routes),
}
}
fn grants(allowed: Option<&[&str]>, denied: &[&str]) -> PassthroughGrants {
PassthroughGrants {
allowed: allowed.map(list),
denied: Some(list(denied)),
}
}
#[rstest]
#[case::team_grant(grants(Some(&["/openai"]), &[]), &[], PassthroughAccess::Allowed)]
#[case::jwt_team_route_grant(grants(None, &[]), &["/openai/*"], PassthroughAccess::Allowed)]
#[case::nothing_grants(grants(None, &[]), &[], PassthroughAccess::Denied { denied_route: None })]
#[case::deny_beats_grant(
grants(Some(&["/openai"]), &["/openai/*"]),
&["/openai/*"],
PassthroughAccess::Denied { denied_route: Some("/openai/*".to_owned()) }
)]
fn team_access(
#[case] team: PassthroughGrants,
#[case] jwt_team_allowed_routes: &[&str],
#[case] expected: PassthroughAccess,
) {
let (forwarded, allowed) = (owned(&["/openai/x"]), owned(jwt_team_allowed_routes));
let groups = route_groups();
assert_eq!(
request(&forwarded, groups.allowlist(&allowed)).team_access(&team),
expected
);
}
#[test]
fn routes_that_enforce_no_auth_are_always_allowed() {
let request = PassthroughRequest {
route: "/openai/x",
..PassthroughRequest::default()
};
assert_eq!(
request.team_access(&grants(None, &["/"])),
PassthroughAccess::Allowed
);
}
}

View file

@ -0,0 +1,73 @@
use super::RouteSet;
/// The `LiteLLMRoutes` groups the core's own rules read. The host sends every group, and the rest
/// are reached only by name from an operator's `allowed_routes`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::VariantArray)]
pub enum RouteGroup {
#[strum(serialize = "public_routes")]
Public,
#[strum(serialize = "master_key_only_routes")]
MasterKeyOnly,
#[strum(serialize = "mcp_inference_routes")]
McpInference,
#[strum(serialize = "agent_inference_routes")]
AgentInference,
}
pub(crate) const MANAGED_DISCOVERY_ROUTES: &[&str] = &["/agents", "/v1/agents"];
pub(crate) const MANAGED_REALTIME_ROUTES: &[&str] = &[
"/realtime",
"/v1/realtime",
"/openai/v1/realtime",
"/realtime/translations",
"/v1/realtime/translations",
];
pub(crate) const MANAGED_LIVE_ROUTES: &[&str] = &[
"/live/sessions",
"/v1/live/sessions",
"/openai/v1/live/sessions",
];
pub(crate) const MANAGED_MODEL_PREFIXES: &[&str] = &["", "/v1"];
pub(crate) const MANAGED_MODEL_OPERATIONS: &[&str] = &[
"chat/completions",
"completions",
"embeddings",
"responses",
"messages",
"messages/count_tokens",
"images/generations",
"images/edits",
"audio/transcriptions",
"audio/speech",
"moderations",
"rerank",
"decisions",
"systemone",
"extraction",
"ocr",
];
pub(crate) const MANAGED_MODEL_EXTRA_ROUTES: &[&str] = &[
"/openai/v1/responses",
"/v2/rerank",
"/claude_code_gateway/v1/messages",
"/claude_code_gateway/v1/messages/count_tokens",
"/cursor/chat/completions",
];
pub(crate) static MANAGED_MODEL_PATHS: RouteSet = RouteSet::new(&[
"/engines/{model:path}/chat/completions",
"/engines/{model:path}/completions",
"/engines/{model:path}/embeddings",
"/openai/deployments/{model:path}/chat/completions",
"/openai/deployments/{model:path}/completions",
"/openai/deployments/{model:path}/embeddings",
"/openai/deployments/{model:path}/images/generations",
"/openai/deployments/{model:path}/images/edits",
"/v1beta/models/{model_name:path}:countTokens",
"/v1beta/models/{model_name:path}:generateContent",
"/v1beta/models/{model_name:path}:streamGenerateContent",
"/models/{model_name:path}:countTokens",
"/models/{model_name:path}:generateContent",
"/models/{model_name:path}:streamGenerateContent",
]);
/// MCP inference routes a managed agent may call; the OAuth token endpoints are not among them.
pub(crate) const MANAGED_MCP_EXCLUDED: &[&str] = &["/token", "/introspect"];

View file

@ -0,0 +1,173 @@
use std::sync::OnceLock;
use litellm_starlette::{Convertor, Param, RouteTemplate};
/// A route template matched as `RouteChecks._route_matches_pattern` matches it: every parameter
/// takes one or more characters up to a `/`, whatever its convertor, and a `{name:path}` one or
/// more of any character, or of any but `:` when the template has a `:` after it. Literal text is
/// compared exactly.
#[derive(Debug)]
pub(super) struct Template(RouteTemplate);
impl Template {
pub(super) fn compile(template: &str) -> Self {
Self(RouteTemplate::parse(template))
}
pub(super) fn matches(&self, route: &str) -> bool {
self.0.matches_with(route, |param, rest| {
let limit = match param {
Param {
convertor: Convertor::Path,
colon_after: true,
} => rest.find(':').unwrap_or(rest.len()),
Param {
convertor: Convertor::Path,
colon_after: false,
} => rest.len(),
Param { .. } => rest.find('/').unwrap_or(rest.len()),
};
Box::new(
rest.char_indices()
.map(|(index, _)| index)
.chain([rest.len()])
.filter(move |index| *index >= 1 && *index <= limit),
)
})
}
/// The route matches the regex starlette's `compile_path` builds from the template, the
/// matcher `_allowed_routes_check` reads route groups with.
pub(super) fn matches_compiled(&self, route: &str) -> bool {
self.0.matches(route)
}
}
/// Route templates named in the core, compiled on first use.
#[derive(Debug)]
pub(crate) struct RouteSet {
templates: &'static [&'static str],
compiled: OnceLock<Box<[Template]>>,
}
impl RouteSet {
pub(crate) const fn new(templates: &'static [&'static str]) -> Self {
Self {
templates,
compiled: OnceLock::new(),
}
}
pub(crate) fn matches(&self, route: &str) -> bool {
self.compiled
.get_or_init(|| {
self.templates
.iter()
.map(|template| Template::compile(template))
.collect()
})
.iter()
.any(|template| template.matches(route))
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::exact("/models", "/models", true)]
#[case::exact_is_whole_route("/models", "/models/x", false)]
#[case::segment("/files/{file_id}", "/files/abc", true)]
#[case::segment_needs_a_char("/files/{file_id}/content", "/files//content", false)]
#[case::segment_stops_at_slash("/files/{file_id}", "/files/a/b", false)]
#[case::path_spans_slashes(
"/v1beta/models/{model_name:path}:generateContent",
"/v1beta/models/a/b:generateContent",
true
)]
#[case::path_needs_a_char("/engines/{model:path}/completions", "/engines//completions", false)]
#[case::path_spans_newlines(
"/engines/{model:path}/chat/completions",
"/engines/a\nb/chat/completions",
true
)]
#[case::path_takes_colons_without_one_after(
"/v1/mcp/server/{path:path}",
"/v1/mcp/server/a:b",
true
)]
#[case::path_stops_before_the_templates_colon(
"/models/{model_name:path}:generateContent",
"/models/a:b:generateContent",
false
)]
#[case::path_never_swallows_another_verb(
"/models/{model_name:path}:countTokens",
"/models/a:generateContent:countTokens",
false
)]
#[case::path_before_colon_needs_a_char(
"/models/{model_name:path}:generateContent",
"/models/:generateContent",
false
)]
#[case::literal_after_param(
"/claude-code/plugins/{plugin_name}/archive/{digest}.zip",
"/claude-code/plugins/p/archive/d.zip",
true
)]
#[case::dot_is_literal(
"/claude-code/plugins/{plugin_name}/archive/{digest}.zip",
"/claude-code/plugins/p/archive/dxzip",
false
)]
#[case::question_mark_is_literal("/realtime?{model}", "/realtime?gpt", true)]
#[case::trailing_newline("/models", "/models\n", true)]
#[case::star_is_literal_in_templates("/a/*", "/a/b", false)]
#[case::host_template_drops_port("chat_completion:80", "chat_completion", true)]
#[case::int_param_is_one_segment("/custom/{item_id:int}", "/custom/5", true)]
#[case::int_param_takes_any_segment("/custom/{item_id:int}", "/custom/abc", true)]
#[case::int_param_stops_at_slash("/custom/{item_id:int}", "/custom/5/x", false)]
#[case::float_param_is_one_segment("/x/{value:float}/y", "/x/1.5/y", true)]
#[case::uuid_param_is_one_segment(
"/x/{id:uuid}",
"/x/1b4e28ba-2fa1-11d2-883f-0016d3cca427",
true
)]
#[case::one_trailing_newline_only("/models", "/models\n\n", false)]
#[case::segment_takes_non_ascii("/files/{file_id}", "/files/\u{e9}", true)]
#[case::two_segments("/a/{x}/b/{y}", "/a/1/b/2", true)]
#[case::adjacent_params_need_a_char_each("/a/{x}{y}", "/a/b", false)]
#[case::adjacent_params_split("/a/{x}{y}", "/a/bc", true)]
#[case::int_param_backtracks_for_a_literal("/x/{id:int}5", "/x/125", true)]
#[case::two_path_params_split("/{a:path}/{b:path}/end", "/x/y/z/end", true)]
#[case::two_path_params_need_the_suffix("/{a:path}/{b:path}/end", "/x/y/z/ends", false)]
#[case::two_path_params_need_a_char_each("/{a:path}/{b:path}/end", "//end", false)]
fn templates_match_like_route_checks(
#[case] template: &str,
#[case] route: &str,
#[case] expected: bool,
) {
assert_eq!(Template::compile(template).matches(route), expected);
}
/// Eight `path` parameters against a route that never ends in `/end`, which trying every
/// split one at a time never finishes.
#[test]
fn many_path_params_answer_in_polynomial_time() {
let params: Vec<String> = (0..8).map(|index| format!("{{p{index}:path}}")).collect();
let template = Template::compile(&format!("/{}/end", params.join("/")));
let route = format!("/{}", "x/".repeat(60));
let (sender, receiver) = std::sync::mpsc::channel();
std::thread::spawn(move || sender.send(template.matches(&route)));
assert_eq!(
receiver
.recv_timeout(std::time::Duration::from_secs(10))
.ok(),
Some(false)
);
}
}

View file

@ -0,0 +1,50 @@
use crate::routes::RouteGroups;
pub fn route_groups() -> RouteGroups {
RouteGroups::new(route_table()).expect("the stand-in table holds every group the core reads")
}
/// A small stand-in for the `LiteLLMRoutes` table the Python host sends.
pub fn route_table() -> Vec<(String, Vec<String>)> {
let group = |name: &str, routes: &[&str]| {
(
name.to_owned(),
routes.iter().map(|route| (*route).to_owned()).collect(),
)
};
vec![
group("public_routes", &["/", "/health/liveliness"]),
group(
"master_key_only_routes",
&[
"/global/spend/reset",
"/memory-usage-in-mem-cache",
"/memory-usage-in-mem-cache-items",
],
),
group(
"openai_routes",
&[
"/chat/completions",
"/v1/chat/completions",
"/engines/{model}/chat/completions",
"/openai/deployments/{model}/chat/completions",
],
),
group("info_routes", &["/key/info", "/user/info", "/team/list"]),
group(
"mcp_routes",
&["/mcp/tools/list", "/mcp/tools/call", "/token"],
),
group(
"mcp_inference_routes",
&["/mcp/{subpath}", "/mcp/tools/call", "/token", "/introspect"],
),
group("agent_inference_routes", &["/a2a/{agent_id}/message/send"]),
group("spend_tracking_routes", &["/spend/logs"]),
group(
"google_routes",
&["/v1beta/models/{model_name:path}:generateContent"],
),
]
}

View file

@ -20,6 +20,7 @@ futures-util.workspace = true
litellm-gateway-auth.workspace = true
litellm-gateway-ui.workspace = true
litellm-config.workspace = true
litellm-config-load.workspace = true
litellm-core-utils.workspace = true
litellm-error.workspace = true
litellm-circuit-breaker.workspace = true

View file

@ -11,7 +11,7 @@ pub enum Error {
#[error(transparent)]
Mcp(#[from] litellm_gateway_mcp::ConnectError),
#[error(transparent)]
Config(#[from] litellm_config::Error),
Config(#[from] litellm_config_load::Error),
#[error(transparent)]
Redis(#[from] litellm_redis_resolve::Error),
#[error("Redis setting {setting} is not plain data")]

View file

@ -3,7 +3,6 @@ use std::{
time::{SystemTime, UNIX_EPOCH},
};
use litellm_config::Config;
use litellm_gateway_auth::UiBackend;
use litellm_tracing::{Level, Logger, Metadata, Record, Sink};
use serde_json::json;
@ -58,7 +57,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
let settings: Settings = settings::from_iter(std::env::vars_os())?;
let level = settings.log_level();
Logger::new(StderrSink { level }).install_global()?;
let config = Config::load(settings.litellm_config)?;
let config = litellm_config_load::load(settings.litellm_config)?;
let inference = litellm_gateway::build_inference(&config)?;
let _redis =
litellm_gateway::build_redis(&config, &inference.resources.pool, &inference.http).await?;

View file

@ -65,7 +65,11 @@ litellm-secrets-types.workspace = true
litellm-llms-types.workspace = true
litellm-router-types.workspace = true
litellm-router-core.workspace = true
litellm-config.workspace = true
litellm-gateway-auth-core.workspace = true
http.workspace = true
litellm-host-python.workspace = true
parking_lot.workspace = true
litellm-model-catalog.workspace = true
litellm-model-catalog-refresh.workspace = true
litellm-token-counter = { path = "../token-counter", default-features = false }

View file

@ -0,0 +1,8 @@
use http::Method;
use pyo3::{exceptions::PyValueError, prelude::*};
/// Upper-cased, as `RouteChecks.get_request_method` hands it over.
pub(crate) fn parse_method(text: &str) -> PyResult<Method> {
Method::from_bytes(text.to_ascii_uppercase().as_bytes())
.map_err(|error| PyValueError::new_err(format!("invalid request method {text:?}: {error}")))
}

View file

@ -0,0 +1,9 @@
mod convert;
mod passthrough;
mod route_groups;
pub(crate) use passthrough::{
auth_allowed_routes_grant, auth_first_denied_route, auth_jwt_team_routes_grant,
auth_managed_agent_route_allowed, auth_managed_route, auth_route_matches_denied,
auth_route_matches_prefix, auth_route_matches_wildcard,
};

View file

@ -0,0 +1,97 @@
use litellm_config::RouteList;
use litellm_gateway_auth_core::{
matches_wildcard,
routes::passthrough::{self, AllowedPassthrough, DeniedPassthrough, ForwardedRoutes},
};
use litellm_host_python::from_py;
use pyo3::prelude::*;
use super::{convert::parse_method, route_groups::route_groups};
#[pyfunction]
pub(crate) fn auth_route_matches_wildcard(route: &str, pattern: &str) -> bool {
matches_wildcard(route, pattern)
}
#[pyfunction]
pub(crate) fn auth_route_matches_prefix(route: &str, allowed_route: &str) -> bool {
passthrough::matches_prefix(route, allowed_route)
}
#[pyfunction]
pub(crate) fn auth_route_matches_denied(route: &str, denied_route: &str) -> bool {
passthrough::matches_denied(route, denied_route)
}
/// `denied_sources` are the raw `denied_passthrough_routes` metadata values, key first.
#[pyfunction]
pub(crate) fn auth_first_denied_route(
forwarded_routes: Vec<String>,
denied_sources: Vec<Bound<'_, PyAny>>,
) -> PyResult<Option<String>> {
let denied = denied_sources
.iter()
.map(route_list)
.filter_map(Result::transpose)
.collect::<PyResult<Vec<_>>>()?;
Ok(DeniedPassthrough(&denied)
.first_match(ForwardedRoutes(&forwarded_routes))
.map(str::to_owned))
}
#[pyfunction]
pub(crate) fn auth_jwt_team_routes_grant(
py: Python<'_>,
route: &str,
#[pyo3(from_py_with = route_list)] team_allowed_routes: Option<RouteList>,
) -> PyResult<bool> {
let entries = team_allowed_routes
.as_ref()
.map(RouteList::items)
.unwrap_or_default();
Ok(route_groups(py)?
.allowlist(entries)
.grants_pass_through(route))
}
/// A raw `allowed_passthrough_routes` or `denied_passthrough_routes` metadata value. A falsy one
/// is absent, as `metadata.get(...) or ...` reads it. One that does not read as a route list is
/// `Malformed`, so it denies instead of failing the request.
pub(crate) fn route_list(value: &Bound<'_, PyAny>) -> PyResult<Option<RouteList>> {
Ok(value
.is_truthy()?
.then(|| from_py(value).unwrap_or(RouteList::Malformed)))
}
/// `key` and `team` are the raw `allowed_passthrough_routes` metadata values.
#[pyfunction]
#[pyo3(signature = (route, *, key, team))]
pub(crate) fn auth_allowed_routes_grant(
route: &str,
#[pyo3(from_py_with = route_list)] key: Option<RouteList>,
#[pyo3(from_py_with = route_list)] team: Option<RouteList>,
) -> bool {
AllowedPassthrough {
key: key.as_ref(),
team: team.as_ref(),
}
.grants(route)
}
/// The kind of route a managed agent identity calls: `discovery`, `realtime`, `live`, `model`,
/// `tool` or `other`.
#[pyfunction]
pub(crate) fn auth_managed_route(py: Python<'_>, route: &str) -> PyResult<&'static str> {
Ok(route_groups(py)?.managed_route(route).into())
}
#[pyfunction]
#[pyo3(signature = (route, method = None))]
pub(crate) fn auth_managed_agent_route_allowed(
py: Python<'_>,
route: &str,
method: Option<&str>,
) -> PyResult<bool> {
let method = method.map(parse_method).transpose()?;
Ok(route_groups(py)?.managed_agent_route_allowed(route, method.as_ref()))
}

View file

@ -0,0 +1,34 @@
use litellm_gateway_auth_core::RouteGroups;
use parking_lot::Mutex;
use pyo3::{exceptions::PyRuntimeError, prelude::*};
const MODULE: &str = "litellm.rust_bridge.host.route_groups";
/// The groups last built, beside the `route_groups()` value they were built from.
static BUILT: Mutex<Option<(Py<PyAny>, RouteGroups)>> = Mutex::new(None);
/// `route_groups()` as the core reads it. `LiteLLMRoutes.openai_routes` grows while the proxy runs,
/// as each auth-enforced pass-through endpoint registers, so the groups are rebuilt whenever the
/// Python value differs from the one they were built from.
pub(crate) fn route_groups(py: Python<'_>) -> PyResult<RouteGroups> {
let current = py.import(MODULE)?.call_method0("route_groups")?;
let cached = BUILT
.lock()
.as_ref()
.map(|(source, groups)| (source.clone_ref(py), groups.clone()));
if let Some((source, groups)) = cached
&& source.bind(py).eq(&current)?
{
return Ok(groups);
}
let groups = RouteGroups::new(
current
.call_method0("items")?
.try_iter()?
.map(|item| item?.extract::<(String, Vec<String>)>())
.collect::<PyResult<Vec<_>>>()?,
)
.map_err(|missing| PyRuntimeError::new_err(missing.to_string()))?;
*BUILT.lock() = Some((current.unbind(), groups.clone()));
Ok(groups)
}

View file

@ -1,3 +1,4 @@
mod auth;
mod budgets;
mod cache;
mod callable;
@ -27,6 +28,12 @@ mod _native {
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
#[pymodule_export]
use crate::auth::{
auth_allowed_routes_grant, auth_first_denied_route, auth_jwt_team_routes_grant,
auth_managed_agent_route_allowed, auth_managed_route, auth_route_matches_denied,
auth_route_matches_prefix, auth_route_matches_wildcard,
};
#[pymodule_export]
use crate::budgets::NativeBudgets;
#[pymodule_export]
@ -215,6 +222,14 @@ mod tests {
"NativeBudgets",
"budget_check_decision",
"NativeS3Log",
"auth_allowed_routes_grant",
"auth_first_denied_route",
"auth_jwt_team_routes_grant",
"auth_managed_agent_route_allowed",
"auth_managed_route",
"auth_route_matches_denied",
"auth_route_matches_prefix",
"auth_route_matches_wildcard",
];
#[cfg(feature = "huggingface")]
expected.push("HuggingFaceEncoding");

View file

@ -23,3 +23,4 @@
- Decoders recurse, so they reject nesting beyond `MAX_DEPTH` for stack safety: deliberately stricter than CPython, whose parser takes ~200 levels and whose unpickler has no limit (pinned as `nested_150`)
- Formatters (`repr`, `json`) are unbounded. Values from the decoders are already capped, a hand-built `Value` is the caller's responsibility
- `literal_eval` must stay linear in depth: `tests/limits.rs` times the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench
- `lenient` holds how Python LiteLLM reads a config or metadata value it never validates, one rule per Python expression (`value is True`, `bool(value)`, a value of the expected shape). A rule never fails, since the Python read never raises. Config types use them as `serde_with` adapters on plain fields, so the gateway's YAML load and python-bridge's `from_py` share one implementation. Each test case takes its expected value from that expression in CPython

View file

@ -12,6 +12,7 @@ num-traits = "0.2"
serde.workspace = true
serde-pickle = "1.2"
serde_json = { workspace = true, features = ["preserve_order"] }
serde_with.workspace = true
thiserror.workspace = true
[dev-dependencies]

View file

@ -0,0 +1,151 @@
//! How Python LiteLLM reads a config or metadata value it never validates: each rule is the Python
//! expression it mirrors, and none of them fails, since the Python read never raises either.
//!
//! | Rule | Python expression |
//! |---|---|
//! | [`IsTrue`] | `value is True` |
//! | [`Truthy`] | `bool(value)` |
//! | [`Lenient`] | the value when it has the expected shape, otherwise ignored |
//!
//! The bool rules are `serde_with` adapters, used as `#[serde_as(as = "IsTrue")]` on a `bool`.
use std::fmt;
use serde::{
Deserialize, Deserializer,
de::{Error, IgnoredAny, MapAccess, SeqAccess, Visitor},
};
use serde_with::DeserializeAs;
/// `value is True`: only a boolean `true` is true. `"true"`, `1` and anything else are false.
pub struct IsTrue;
impl<'de> DeserializeAs<'de, bool> for IsTrue {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<bool, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Raw {
Bool(bool),
Other(IgnoredAny),
}
Ok(matches!(Raw::deserialize(deserializer)?, Raw::Bool(true)))
}
}
/// `bool(value)` for plain data: `None`, `False`, zero, and empty strings and containers are false,
/// as [`crate::truthy::truthy`] reads a [`crate::Value`].
pub struct Truthy;
impl<'de> DeserializeAs<'de, bool> for Truthy {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<bool, D::Error> {
deserializer.deserialize_any(TruthyVisitor)
}
}
struct TruthyVisitor;
impl<'de> Visitor<'de> for TruthyVisitor {
type Value = bool;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("any value")
}
fn visit_bool<E: Error>(self, value: bool) -> Result<bool, E> {
Ok(value)
}
fn visit_i64<E: Error>(self, value: i64) -> Result<bool, E> {
Ok(value != 0)
}
fn visit_u64<E: Error>(self, value: u64) -> Result<bool, E> {
Ok(value != 0)
}
fn visit_i128<E: Error>(self, value: i128) -> Result<bool, E> {
Ok(value != 0)
}
fn visit_u128<E: Error>(self, value: u128) -> Result<bool, E> {
Ok(value != 0)
}
fn visit_f64<E: Error>(self, value: f64) -> Result<bool, E> {
Ok(value != 0.0)
}
fn visit_str<E: Error>(self, value: &str) -> Result<bool, E> {
Ok(!value.is_empty())
}
fn visit_bytes<E: Error>(self, value: &[u8]) -> Result<bool, E> {
Ok(!value.is_empty())
}
fn visit_none<E: Error>(self) -> Result<bool, E> {
Ok(false)
}
fn visit_unit<E: Error>(self) -> Result<bool, E> {
Ok(false)
}
fn visit_some<D: Deserializer<'de>>(self, deserializer: D) -> Result<bool, D::Error> {
deserializer.deserialize_any(self)
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<bool, A::Error> {
let non_empty = seq.next_element::<IgnoredAny>()?.is_some();
while seq.next_element::<IgnoredAny>()?.is_some() {}
Ok(non_empty)
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<bool, A::Error> {
let non_empty = map.next_entry::<IgnoredAny, IgnoredAny>()?.is_some();
while map.next_entry::<IgnoredAny, IgnoredAny>()?.is_some() {}
Ok(non_empty)
}
}
/// A value Python only uses when it has the shape `T`, such as a list of route strings. Any
/// other value is `Malformed` rather than an error, and the caller decides what that grants.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[serde(from = "RawLenient<T>", bound(deserialize = "T: Deserialize<'de>"))]
pub enum Lenient<T> {
Valid(T),
Malformed,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum RawLenient<T> {
Valid(T),
Malformed(IgnoredAny),
}
impl<T> From<RawLenient<T>> for Lenient<T> {
fn from(raw: RawLenient<T>) -> Self {
match raw {
RawLenient::Valid(value) => Self::Valid(value),
RawLenient::Malformed(_) => Self::Malformed,
}
}
}
impl<T> Lenient<T> {
pub fn valid(&self) -> Option<&T> {
match self {
Self::Valid(value) => Some(value),
Self::Malformed => None,
}
}
}
impl<T> Lenient<Box<[T]>> {
/// The list's items, none when malformed.
pub fn items(&self) -> &[T] {
self.valid().map(|items| &**items).unwrap_or_default()
}
}

View file

@ -17,6 +17,7 @@
//! | [`pydantic`] | `TypeAdapter(bool)` and `TypeAdapter(int)` lax validation |
//! | [`urllib`] | `urllib.parse.quote(text, safe=safe)` |
//! | [`datetime`] | `datetime.fromtimestamp(seconds, timezone.utc)` fields and `datetime.fromisoformat(text)` |
//! | [`lenient`] | how Python reads an unvalidated config value: `value is True`, or a value of the expected shape |
//!
//! [`Value`] is the closed data model these formats share. Live Python objects
//! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the
@ -33,6 +34,7 @@ pub mod datetime;
mod error;
pub mod format;
pub mod json;
pub mod lenient;
pub mod literal;
pub mod number;
pub mod pickle;

View file

@ -0,0 +1,72 @@
use litellm_python_compat::lenient::{IsTrue, Lenient, Truthy};
use rstest::rstest;
use serde::Deserialize;
use serde_json::{Value, json};
use serde_with::serde_as;
#[serde_as]
#[derive(Deserialize)]
struct Flag(#[serde_as(as = "IsTrue")] bool);
/// Expected values from CPython: `value is True` for the JSON value loaded with `json.loads`.
#[rstest]
#[case::true_(json!(true), true)]
#[case::false_(json!(false), false)]
#[case::string_true(json!("true"), false)]
#[case::one(json!(1), false)]
#[case::null(json!(null), false)]
#[case::list_with_true(json!([true]), false)]
fn is_true_is_only_the_boolean_true(#[case] value: Value, #[case] expected: bool) {
assert_eq!(serde_json::from_value::<Flag>(value).unwrap().0, expected);
}
#[serde_as]
#[derive(Deserialize)]
struct Bool(#[serde_as(as = "Truthy")] bool);
/// Expected values from CPython: `bool(value)` for the JSON value loaded with `json.loads`.
#[rstest]
#[case::true_(json!(true), true)]
#[case::false_(json!(false), false)]
#[case::zero(json!(0), false)]
#[case::one(json!(1), true)]
#[case::zero_float(json!(0.0), false)]
#[case::half(json!(0.5), true)]
#[case::empty_string(json!(""), false)]
#[case::string_false(json!("false"), true)]
#[case::null(json!(null), false)]
#[case::empty_list(json!([]), false)]
#[case::list_of_false(json!([false]), true)]
#[case::empty_mapping(json!({}), false)]
#[case::mapping(json!({"a": null}), true)]
fn truthy_is_python_bool(#[case] value: Value, #[case] expected: bool) {
assert_eq!(serde_json::from_value::<Bool>(value).unwrap().0, expected);
}
#[rstest]
#[case::list(json!(["/a", "/b/*"]), Lenient::Valid(["/a".to_owned(), "/b/*".to_owned()].into()))]
#[case::empty(json!([]), Lenient::Valid([].into()))]
#[case::bare_string(json!("/a/*"), Lenient::Malformed)]
#[case::number(json!(5), Lenient::Malformed)]
#[case::mapping(json!({"a": 1}), Lenient::Malformed)]
#[case::non_string_entry(json!(["/a", 5, "/b/*"]), Lenient::Malformed)]
#[case::null_entry(json!(["/a", null]), Lenient::Malformed)]
fn a_value_of_another_shape_is_malformed(
#[case] value: Value,
#[case] expected: Lenient<Box<[String]>>,
) {
assert_eq!(
serde_json::from_value::<Lenient<Box<[String]>>>(value).unwrap(),
expected
);
}
#[rstest]
#[case::listed(Lenient::Valid(["/a".to_owned()].into()), &["/a"][..])]
#[case::malformed(Lenient::Malformed, &[][..])]
fn items_of_a_malformed_list_are_none(
#[case] list: Lenient<Box<[String]>>,
#[case] expected: &[&str],
) {
assert_eq!(list.items(), expected);
}

View file

@ -0,0 +1,7 @@
# starlette
Starlette's route template syntax, for route templates Python LiteLLM registers with FastAPI and matches with starlette, such as `LiteLLMRoutes` groups and pass-through `path`s. A crate that reads one of those templates depends on this crate and never parses `{name:convertor}` itself
- `RouteTemplate::parse` follows `starlette.routing.compile_path`: `PARAM_REGEX` names, the five `CONVERTOR_TYPES`, and the port a host template drops. It never fails, where starlette raises on an unknown convertor, because starlette refuses such a route at registration, so no live route has one. It reads as `str`
- `RouteTemplate::matches` is the regex `compile_path` builds. Other matching rules LiteLLM writes over the same templates, such as `RouteChecks._route_matches_pattern`, belong to their callers, which pass what each parameter takes to `matches_with` rather than walking `parts()` themselves, so every matcher stays polynomial however many parameters a template has
- Every case in `tests/` takes its expected value from running `bool(compile_path(template)[0].match(route))` on starlette, never from reasoning. Run it before adding or changing a case, and update the version the tests cite when the proxy's starlette changes a convertor

View file

@ -0,0 +1,13 @@
[package]
name = "litellm-starlette"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
serde.workspace = true
strum.workspace = true
[dev-dependencies]
rstest.workspace = true

View file

@ -0,0 +1,5 @@
#![forbid(unsafe_code)]
mod template;
pub use template::{Convertor, Param, RouteTemplate, TemplatePart};

View file

@ -0,0 +1,209 @@
use std::collections::BTreeSet;
/// A starlette path convertor, by the name a template spells it with. `{name}` is `Str`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString)]
pub enum Convertor {
#[strum(serialize = "str")]
Str,
#[strum(serialize = "path")]
Path,
#[strum(serialize = "int")]
Int,
#[strum(serialize = "float")]
Float,
#[strum(serialize = "uuid")]
Uuid,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Param {
pub convertor: Convertor,
/// The template has a `:` anywhere after this parameter, as the Google
/// `{model:path}:generateContent` routes do.
pub colon_after: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TemplatePart {
Literal(Box<str>),
Param(Param),
}
/// A route template such as `/threads/{thread_id}` or a pass-through `path`. Every string parses:
/// a convertor starlette does not know reads as `Str`, and a host template such as
/// `chat_completion:80` drops its port.
#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize)]
#[serde(from = "String")]
pub struct RouteTemplate(Box<[TemplatePart]>);
impl From<String> for RouteTemplate {
fn from(template: String) -> Self {
Self::parse(&template)
}
}
impl RouteTemplate {
pub fn parse(template: &str) -> Self {
let is_host = !template.starts_with('/');
let mut parts = Vec::new();
let mut rest = template;
while let Some((before, convertor, after)) = next_param(rest) {
parts.push(TemplatePart::Literal(before.into()));
parts.push(TemplatePart::Param(Param {
convertor: convertor
.and_then(|name| name.parse().ok())
.unwrap_or(Convertor::Str),
colon_after: after.contains(':'),
}));
rest = after;
}
let tail = if is_host {
rest.split(':').next().unwrap_or_default()
} else {
rest
};
parts.push(TemplatePart::Literal(tail.into()));
Self(
parts
.into_iter()
.filter(|part| !matches!(part, TemplatePart::Literal(text) if text.is_empty()))
.collect(),
)
}
pub fn parts(&self) -> &[TemplatePart] {
&self.0
}
/// The route matches the regex `compile_path` builds from the template. Each convertor takes
/// what its starlette regex takes: `str` one or more of anything but `/`, `path` zero or more
/// of anything but a newline, `int` digits, `float` digits with an optional fraction, and
/// `uuid` 32 hex digits with optional dashes between the groups.
pub fn matches(&self, route: &str) -> bool {
self.matches_with(route, |param, rest| convertor_ends(param.convertor, rest))
}
/// The route matches with literal text compared exactly and each parameter taking any length
/// `param_ends` yields for the rest of the route. Like `re.match` against a pattern ending in
/// `$`, a trailing newline is allowed.
pub fn matches_with(
&self,
route: &str,
param_ends: impl for<'r> Fn(Param, &'r str) -> Box<dyn Iterator<Item = usize> + 'r>,
) -> bool {
walk(&self.0, route, &param_ends)
}
}
fn is_identifier(text: &str) -> bool {
let mut chars = text.chars();
chars
.next()
.is_some_and(|first| first.is_ascii_alphabetic() || first == '_')
&& chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
}
/// The next `{name}` or `{name:convertor}`, as starlette's `PARAM_REGEX` finds it: the text before
/// it, its convertor and the text after it.
fn next_param(text: &str) -> Option<(&str, Option<&str>, &str)> {
text.match_indices('{').find_map(|(start, _)| {
let end = start + text[start..].find('}')?;
let inner = &text[start + 1..end];
let (name, convertor) = match inner.split_once(':') {
Some((name, convertor)) => (name, Some(convertor)),
None => (inner, None),
};
(is_identifier(name) && convertor.is_none_or(is_identifier))
.then(|| (&text[..start], convertor, &text[end + 1..]))
})
}
fn is_end(rest: &str) -> bool {
rest.is_empty() || rest == "\n"
}
/// Literals never branch, so they are stripped in place. From the first parameter on, the walk
/// keeps the set of offsets the parts so far can end at instead of trying each split in turn,
/// which keeps it at parts x route length x choices per parameter however many parameters a
/// template has.
fn walk(
parts: &[TemplatePart],
route: &str,
param_ends: &impl for<'r> Fn(Param, &'r str) -> Box<dyn Iterator<Item = usize> + 'r>,
) -> bool {
match parts.split_first() {
None => is_end(route),
Some((TemplatePart::Literal(literal), rest)) => route
.strip_prefix(&**literal)
.is_some_and(|after| walk(rest, after, param_ends)),
Some((TemplatePart::Param(_), _)) => parts
.iter()
.try_fold(BTreeSet::from([0]), |starts, part| {
let ends: BTreeSet<usize> = match part {
TemplatePart::Literal(literal) => starts
.into_iter()
.filter(|&start| route[start..].starts_with(&**literal))
.map(|start| start + literal.len())
.collect(),
TemplatePart::Param(param) => starts
.into_iter()
.flat_map(|start| {
param_ends(*param, &route[start..]).map(move |len| start + len)
})
.collect(),
};
(!ends.is_empty()).then_some(ends)
})
.is_some_and(|ends| ends.into_iter().any(|end| is_end(&route[end..]))),
}
}
/// Every length of `route`'s start the convertor's regex can match.
fn convertor_ends(convertor: Convertor, route: &str) -> Box<dyn Iterator<Item = usize> + '_> {
let boundaries = move |limit: usize, min: usize| {
route
.char_indices()
.map(|(index, _)| index)
.chain([route.len()])
.filter(move |index| *index >= min && *index <= limit)
};
let digits = |text: &str| text.bytes().take_while(u8::is_ascii_digit).count();
match convertor {
Convertor::Str => Box::new(boundaries(route.find('/').unwrap_or(route.len()), 1)),
Convertor::Path => Box::new(boundaries(route.find('\n').unwrap_or(route.len()), 0)),
Convertor::Int => Box::new(1..=digits(route)),
Convertor::Float => {
let whole = digits(route);
let fraction = route[whole..]
.strip_prefix('.')
.map(digits)
.unwrap_or_default();
let fractions = (whole > 0)
.then_some(1..=fraction)
.into_iter()
.flatten()
.map(move |digits| whole + 1 + digits);
Box::new((1..=whole).chain(fractions))
}
Convertor::Uuid => Box::new(uuid_len(route).into_iter()),
}
}
/// `[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}`. A dash can
/// never be a hex digit, so the match length is unique.
fn uuid_len(route: &str) -> Option<usize> {
let bytes = route.as_bytes();
[8, 4, 4, 4, 12]
.iter()
.enumerate()
.try_fold(0, |at, (group, &width)| {
let at = match (group > 0, bytes.get(at)) {
(true, Some(b'-')) => at + 1,
_ => at,
};
bytes
.get(at..at + width)
.filter(|hex| hex.iter().all(u8::is_ascii_hexdigit))
.map(|_| at + width)
})
}

View file

@ -0,0 +1,114 @@
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use litellm_starlette::{Convertor, Param, RouteTemplate, TemplatePart};
use rstest::rstest;
fn literal(text: &str) -> TemplatePart {
TemplatePart::Literal(text.into())
}
fn param(convertor: Convertor, colon_after: bool) -> TemplatePart {
TemplatePart::Param(Param {
convertor,
colon_after,
})
}
#[rstest]
#[case::plain("/models", vec![literal("/models")])]
#[case::segment("/files/{file_id}", vec![literal("/files/"), param(Convertor::Str, false)])]
#[case::int("/custom/{item_id:int}", vec![literal("/custom/"), param(Convertor::Int, false)])]
#[case::float("/x/{v:float}/y", vec![literal("/x/"), param(Convertor::Float, false), literal("/y")])]
#[case::uuid("/x/{id:uuid}", vec![literal("/x/"), param(Convertor::Uuid, false)])]
#[case::unknown_convertor_is_str("/x/{id:slug}", vec![literal("/x/"), param(Convertor::Str, false)])]
#[case::path_before_colon(
"/models/{m:path}:generateContent",
vec![literal("/models/"), param(Convertor::Path, true), literal(":generateContent")]
)]
#[case::colon_after_counts_a_later_convertor(
"/a/{x:path}{y:int}",
vec![literal("/a/"), param(Convertor::Path, true), param(Convertor::Int, false)]
)]
#[case::not_a_param("/x/{1}", vec![literal("/x/{1}")])]
#[case::host_drops_port("chat_completion:80", vec![literal("chat_completion")])]
fn templates_parse(#[case] template: &str, #[case] expected: Vec<TemplatePart>) {
assert_eq!(RouteTemplate::parse(template).parts(), expected.as_slice());
}
/// Expected values from starlette 1.3.1: `bool(compile_path(template)[0].match(route))`.
#[rstest]
#[case::exact("/models", "/models", true)]
#[case::trailing_newline("/models", "/models\n", true)]
#[case::one_trailing_newline_only("/models", "/models\n\n", false)]
#[case::str_needs_a_char("/files/{file_id}", "/files/", false)]
#[case::str_stops_at_slash("/files/{file_id}", "/files/a/b", false)]
#[case::str_takes_a_newline("/files/{file_id}", "/files/a\nb", true)]
#[case::str_takes_non_ascii("/files/{file_id}", "/files/\u{e9}", true)]
#[case::two_str_params("/a/{x}/b/{y}", "/a/1/b/2", true)]
#[case::adjacent_str_params_need_a_char_each("/a/{x}{y}", "/a/b", false)]
#[case::adjacent_str_params_split("/a/{x}{y}", "/a/bc", true)]
#[case::path_spans_slashes("/engines/{m:path}/completions", "/engines/a/b/completions", true)]
#[case::path_may_be_empty(
"/v1beta/models/{m:path}:generateContent",
"/v1beta/models/:generateContent",
true
)]
#[case::path_swallows_colons(
"/v1beta/models/{m:path}:generateContent",
"/v1beta/models/a:b:generateContent",
true
)]
#[case::path_stops_at_a_newline(
"/engines/{m:path}/completions",
"/engines/a\nb/completions",
false
)]
#[case::two_path_params_split("/{a:path}/{b:path}/end", "/x/y/z/end", true)]
#[case::two_path_params_need_the_suffix("/{a:path}/{b:path}/end", "/x/y/z/ends", false)]
#[case::two_path_params_need_both_slashes("/{a:path}/{b:path}/end", "//end", false)]
#[case::int_takes_digits("/x/{id:int}", "/x/42", true)]
#[case::int_needs_a_digit("/x/{id:int}", "/x/", false)]
#[case::int_refuses_letters("/x/{id:int}", "/x/4a", false)]
#[case::int_refuses_non_ascii_digits("/x/{id:int}", "/x/\u{663}", false)]
#[case::int_backtracks_for_a_literal("/x/{id:int}5", "/x/125", true)]
#[case::float_with_fraction("/x/{v:float}", "/x/1.5", true)]
#[case::float_without_fraction("/x/{v:float}", "/x/1", true)]
#[case::float_needs_whole_digits("/x/{v:float}", "/x/.5", false)]
#[case::float_needs_fraction_digits("/x/{v:float}", "/x/1.", false)]
#[case::float_takes_one_fraction("/x/{v:float}/y", "/x/1.5.5/y", false)]
#[case::float_backtracks_inside_the_fraction("/x/{v:float}0", "/x/1.50", true)]
#[case::float_leaves_the_fraction_to_a_literal("/x/{v:float}.5", "/x/1.5", true)]
#[case::uuid_with_dashes("/x/{id:uuid}", "/x/1b4e28ba-2fa1-11d2-883f-0016D3CCA427", true)]
#[case::uuid_without_dashes("/x/{id:uuid}", "/x/1b4e28ba2fa111d2883f0016d3cca427", true)]
#[case::uuid_too_short("/x/{id:uuid}", "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca42", false)]
#[case::uuid_too_long("/x/{id:uuid}", "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca427a", false)]
#[case::uuid_then_literal("/x/{id:uuid}/y", "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca427/y", true)]
#[case::dot_is_literal("/a/{x}.zip", "/a/bxzip", false)]
fn templates_match_like_compile_path(
#[case] template: &str,
#[case] route: &str,
#[case] expected: bool,
) {
assert_eq!(RouteTemplate::parse(template).matches(route), expected);
}
fn answer_within(budget: Duration, check: impl FnOnce() -> bool + Send + 'static) -> Option<bool> {
let (sender, receiver) = mpsc::channel();
thread::spawn(move || sender.send(check()));
receiver.recv_timeout(budget).ok()
}
/// A pass-through `path` with eight `path` parameters against a route that never ends in `/end`.
/// Trying every split one at a time never finishes. Expected value from starlette 1.3.1.
#[test]
fn many_path_params_answer_in_polynomial_time() {
let params: Vec<String> = (0..8).map(|index| format!("{{p{index}:path}}")).collect();
let template = RouteTemplate::parse(&format!("/{}/end", params.join("/")));
let route = format!("/{}", "x/".repeat(60));
assert_eq!(
answer_within(Duration::from_secs(10), move || template.matches(&route)),
Some(false)
);
}

View file

@ -523,6 +523,14 @@ class LocalSizes:
@property
def key_objects(self) -> int: ...
def auth_route_matches_wildcard(route: str, pattern: str) -> bool: ...
def auth_managed_route(route: str) -> Literal["discovery", "realtime", "live", "model", "tool", "other"]: ...
def auth_managed_agent_route_allowed(route: str, method: str | None = None) -> bool: ...
def auth_route_matches_prefix(route: str, allowed_route: str) -> bool: ...
def auth_route_matches_denied(route: str, denied_route: str) -> bool: ...
def auth_first_denied_route(forwarded_routes: Sequence[str], denied_sources: Sequence[object]) -> str | None: ...
def auth_jwt_team_routes_grant(route: str, team_allowed_routes: object) -> bool: ...
def auth_allowed_routes_grant(route: str, *, key: object, team: object) -> bool: ...
@final
class NativeAuthCache:
def __new__(
@ -872,6 +880,14 @@ __all__ = [
"aocr",
"aresponses",
"atranscription",
"auth_allowed_routes_grant",
"auth_first_denied_route",
"auth_jwt_team_routes_grant",
"auth_managed_agent_route_allowed",
"auth_managed_route",
"auth_route_matches_denied",
"auth_route_matches_prefix",
"auth_route_matches_wildcard",
"budget_check_decision",
"bundled_model_cost_map",
"completion",

View file

@ -32,6 +32,7 @@ Python behavior Rust needs but does not own, behind one module per question. Rus
- `redis_events`: reporting native Redis calls and breaker transitions to `ServiceLogging`
- `redis_credentials`: what a caller's redis-py `CredentialProvider` authenticates with
- `model_cost`: where a `register_model` entry lands in the published table and the row it writes
- `route_groups`: the proxy's `LiteLLMRoutes` groups by name, which the auth core reads at settings build
- `routes/<route>.py` holds the questions one route asks, under the same function names on every route (`response`, `map_failure`, `connection_defaults`), so Rust names only the module. A question two routes ask goes in a top-level module here instead
- Each module reuses the Python implementation the Python route runs (`AnthropicModelInfo`, `Logging`). Do not re-derive those rules here or in Rust; one resolver, two callers

View file

@ -0,0 +1,10 @@
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from litellm.proxy._types import LiteLLMRoutes
def route_groups() -> Mapping[str, tuple[str, ...]]:
return MappingProxyType({member.name: tuple(member.value) for member in LiteLLMRoutes})

View file

@ -1884,6 +1884,49 @@ def test_check_passthrough_route_access_empty_list():
assert result is False
@pytest.mark.parametrize(
"key_routes",
[
pytest.param("/custom-endpoint", id="bare_string"),
pytest.param(["/custom-endpoint", None], id="null_entry_after_a_match"),
pytest.param(5, id="number"),
pytest.param(object(), id="unconvertible_object"),
],
)
def test_a_malformed_key_list_grants_nothing_and_shadows_the_team_list(key_routes: object) -> None:
valid_token: Final = UserAPIKeyAuth(
user_id="test_user",
metadata={"allowed_passthrough_routes": key_routes},
team_metadata={"allowed_passthrough_routes": ["/custom-endpoint"]},
)
assert (
RouteChecks.check_passthrough_route_access(route="/custom-endpoint/x", user_api_key_dict=valid_token) is False
)
@pytest.mark.parametrize(
"key_routes",
[
pytest.param([], id="empty_list"),
pytest.param("", id="empty_string"),
pytest.param(0, id="zero"),
pytest.param(False, id="false"),
pytest.param({}, id="empty_dict"),
],
)
def test_an_empty_key_list_falls_back_to_the_team_list(key_routes: object) -> None:
valid_token: Final = UserAPIKeyAuth(
user_id="test_user",
metadata={"allowed_passthrough_routes": key_routes},
team_metadata={"allowed_passthrough_routes": ["/custom-endpoint"]},
)
assert (
RouteChecks.check_passthrough_route_access(route="/custom-endpoint/x", user_api_key_dict=valid_token) is True
)
@pytest.mark.parametrize(
"route",
[
@ -4708,6 +4751,71 @@ def test_denied_passthrough_routes_win_over_allow(
assert f"Matched `{denied_route}` in `denied_passthrough_routes`" in exc_info.value.detail
@pytest.mark.parametrize(
"denied",
[
pytest.param(["/other", None], id="null_entry"),
pytest.param("/other", id="bare_string"),
pytest.param(5, id="number"),
],
)
def test_a_malformed_deny_list_denies_every_route_instead_of_failing(denied: object) -> None:
valid_token: Final = UserAPIKeyAuth(
user_id="test_user",
user_role=LitellmUserRoles.INTERNAL_USER.value,
metadata={"allowed_passthrough_routes": ["/svc"]},
team_metadata={"denied_passthrough_routes": denied},
)
with pytest.raises(HTTPException) as exc_info:
_check_route_with_registered_routes(route="/svc/public", valid_token=valid_token)
assert exc_info.value.status_code == 403
assert "Matched `(not a list of routes)` in `denied_passthrough_routes`" in exc_info.value.detail
@pytest.mark.parametrize(
"denied",
[
pytest.param([], id="empty_list"),
pytest.param("", id="empty_string"),
pytest.param(0, id="zero"),
pytest.param(False, id="false"),
pytest.param({}, id="empty_dict"),
],
)
def test_an_empty_deny_list_denies_nothing(denied: object) -> None:
valid_token: Final = UserAPIKeyAuth(
user_id="test_user",
user_role=LitellmUserRoles.INTERNAL_USER.value,
metadata={"allowed_passthrough_routes": ["/svc"]},
team_metadata={"denied_passthrough_routes": denied},
)
assert (
RouteChecks.matching_denied_passthrough_route(
route="/svc/public", metadata_sources=(valid_token.metadata, valid_token.team_metadata)
)
is None
)
_check_route_with_registered_routes(route="/svc/public", valid_token=valid_token)
@pytest.mark.parametrize(
("team_allowed_routes", "expected"),
[(["/openai/*"], True), (["/openai/*", None], False), ("/openai/*", False)],
ids=["list", "null_entry", "bare_string"],
)
def test_jwt_team_routes_grant_only_from_a_list_of_routes(team_allowed_routes: object, expected: bool) -> None:
assert (
RouteChecks.jwt_team_routes_grant_pass_through(
route="/openai/x",
team_allowed_routes=team_allowed_routes, # pyright: ignore[reportArgumentType] # malformed on purpose
)
is expected
)
@pytest.mark.parametrize(
"route",
["/svc/public", "/svc/administrator", "/anthropic/v1/messages", "/chat/completions"],

View file

@ -0,0 +1,10 @@
from litellm.proxy._types import LiteLLMRoutes
from litellm.rust_bridge.host.route_groups import route_groups
def test_route_groups_hand_rust_every_litellm_route_group_by_name():
groups = route_groups()
assert {name: set(routes) for name, routes in groups.items()} == {
member.name: set(member.value) for member in LiteLLMRoutes
}