diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 57e99edecba..3782b611d1b 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -1,79 +1,16 @@ from collections.abc import Mapping -from itertools import product from types import MappingProxyType from typing import Annotated, Final, Literal from pydantic import Field, TypeAdapter, ValidationError -from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failure +from litellm.rust_bridge._native import auth_managed_agent_route_allowed, auth_managed_route from litellm.types.agents import AgentResponse from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext -_MANAGED_REALTIME_ROUTES: Final = frozenset( - ( - "/realtime", - "/v1/realtime", - "/openai/v1/realtime", - "/realtime/translations", - "/v1/realtime/translations", - ) -) -_MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions")) -_MANAGED_MODEL_ROUTES: Final = frozenset( - f"{prefix}/{operation}" - for prefix, operation in product( - ("", "/v1"), - ( - "chat/completions", - "completions", - "embeddings", - "responses", - "messages", - "messages/count_tokens", - "images/generations", - "images/edits", - "audio/transcriptions", - "audio/speech", - "moderations", - "rerank", - "decisions", - "systemone", - "extraction", - "ocr", - ), - ) -) | frozenset( - ( - "/openai/v1/responses", - "/v2/rerank", - "/claude_code_gateway/v1/messages", - "/claude_code_gateway/v1/messages/count_tokens", - "/cursor/chat/completions", - ) -) -_MANAGED_MODEL_PATHS: Final = ( - "/engines/{model:path}/chat/completions", - "/engines/{model:path}/completions", - "/engines/{model:path}/embeddings", - "/openai/deployments/{model:path}/chat/completions", - "/openai/deployments/{model:path}/completions", - "/openai/deployments/{model:path}/embeddings", - "/openai/deployments/{model:path}/images/generations", - "/openai/deployments/{model:path}/images/edits", - "/v1beta/models/{model_name:path}:countTokens", - "/v1beta/models/{model_name:path}:generateContent", - "/v1beta/models/{model_name:path}:streamGenerateContent", - "/models/{model_name:path}:countTokens", - "/models/{model_name:path}:generateContent", - "/models/{model_name:path}:streamGenerateContent", -) -_MANAGED_MCP_ROUTES: Final = tuple( - route for route in LiteLLMRoutes.mcp_inference_routes.value if route not in ("/token", "/introspect") -) - - _MODEL_ROUTE_KINDS: Final[ Mapping[str, Literal["image_generation", "image_edit", "moderation", "speech", "body", "path"]] ] = MappingProxyType( @@ -94,17 +31,7 @@ _MODEL_ROUTE_KINDS: Final[ def managed_agent_route_allowed(route: str, method: str | None) -> bool: - from litellm.proxy.auth.route_checks import RouteChecks - - if route in ("/agents", "/v1/agents"): - return method in (None, "GET", "HEAD") - if route in _MANAGED_REALTIME_ROUTES or route in _MANAGED_LIVE_ROUTES: - return method in (None, "GET") - if route in _MANAGED_MODEL_ROUTES or RouteChecks.check_route_access(route, _MANAGED_MODEL_PATHS): - return method in (None, "POST") - return RouteChecks.check_route_access(route, _MANAGED_MCP_ROUTES) or RouteChecks.check_route_access( - route, LiteLLMRoutes.agent_inference_routes.value - ) + return auth_managed_agent_route_allowed(route, method) def managed_inference_request( @@ -115,11 +42,8 @@ def managed_inference_request( path_model: object = None, query_model: object = None, ) -> dict[str, object]: - from litellm.proxy.auth.route_checks import RouteChecks - - if route in _MANAGED_LIVE_ROUTES: - return dict(body) - if route in _MANAGED_REALTIME_ROUTES: + managed_route: Final = auth_managed_route(route) + if managed_route == "realtime": from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model model: Final = resolve_realtime_route_model( @@ -130,7 +54,7 @@ def managed_inference_request( AgentIdentityFailure(message="Managed inference requires an explicit or configured model") ) return {**body, "model": model} - if route not in _MANAGED_MODEL_ROUTES and not RouteChecks.check_route_access(route, _MANAGED_MODEL_PATHS): + if managed_route != "model": return dict(body) from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 6116b618c24..de6a6a47fff 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -1,4 +1,3 @@ -import itertools import re from collections.abc import Collection, Iterable, Mapping from typing import Final, cast @@ -18,6 +17,14 @@ from litellm.proxy._types import ( ProxyException, UserAPIKeyAuth, ) +from litellm.rust_bridge._native import ( + auth_allowed_routes_grant, + auth_first_denied_route, + auth_jwt_team_routes_grant, + auth_route_matches_denied, + auth_route_matches_prefix, + auth_route_matches_wildcard, +) from .auth_checks_organization import ( # noqa: F401 # legacy module exports _user_is_org_admin, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -111,6 +118,10 @@ _TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "co _TOKEN_COUNTING_ACTION: Final = "countTokens" +def _allowed_passthrough_routes(metadata: Mapping[str, object] | None) -> object: + return (metadata or {}).get("allowed_passthrough_routes") + + class RouteChecks: @staticmethod def check_route_enabled(route: str) -> None: @@ -659,58 +670,13 @@ class RouteChecks: @staticmethod def route_matches_wildcard_pattern(route: str, pattern: str) -> bool: - """ - Check if route matches the wildcard pattern - - eg. - - pattern: "/scim/v2/*" - route: "/scim/v2/Users" - - returns: True - - pattern: "/scim/v2/*" - route: "/chat/completions" - - returns: False - - - pattern: "/scim/v2/*" - route: "/scim/v2/Users/123" - - returns: True - - """ - if pattern.endswith("*"): - # Get the prefix (everything before the wildcard) - prefix: Final = pattern[:-1] - return route.startswith(prefix) - else: - # If there's no wildcard, the pattern and route should match exactly - return route == pattern + """A trailing `*` matches by prefix, anything else exactly.""" + return auth_route_matches_wildcard(route, pattern) @staticmethod def _route_matches_allowed_route(route: str, allowed_route: str) -> bool: - """ - Check if route matches the allowed_route pattern. - Supports both exact match and prefix match. - - Examples: - - allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-6" -> True (exact match) - - allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-6/v1/chat/completions" -> True (prefix match) - - allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-600" -> False (not a valid prefix) - - Args: - route: The actual route being accessed - allowed_route: The allowed route pattern - - Returns: - bool: True if route matches (exact or prefix), False otherwise - """ - # Exact match - if route == allowed_route: - return True - # Prefix match - ensure we add "/" to prevent false matches like /fake-openai-proxy-600 - if route.startswith(allowed_route + "/"): - return True - return False + """The route itself or anything under it, never a sibling that only shares the prefix.""" + return auth_route_matches_prefix(route, allowed_route) @staticmethod def check_route_access(route: str, allowed_routes: Collection[str]) -> bool: @@ -813,13 +779,16 @@ class RouteChecks: @staticmethod def _route_matches_denied_route(route: str, denied_route: str) -> bool: """A `/` entry denies every route, since every route sits under the root.""" - normalized_denied_route: Final = denied_route.rstrip("/") or "/" - return ( - normalized_denied_route == "/" - or RouteChecks._route_matches_allowed_route(route=route, allowed_route=normalized_denied_route) - or RouteChecks.route_matches_wildcard_pattern(route=route, pattern=denied_route) + return auth_route_matches_denied(route, denied_route) + + @staticmethod + def forwarded_pass_through_routes(route: str) -> tuple[str, ...]: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass-through endpoints import this module + InitPassThroughEndpointHelpers, ) + return InitPassThroughEndpointHelpers.forwarded_routes(route) + @staticmethod def matching_denied_passthrough_route( route: str, metadata_sources: Iterable[Mapping[str, object] | None] @@ -828,32 +797,10 @@ class RouteChecks: First ``denied_passthrough_routes`` entry across ``metadata_sources`` that matches ``route``. Unlike the allowlist (key list, else team list), every source's deny list applies. """ - denied_routes: Final = tuple( - itertools.chain.from_iterable( - cast( # cast-ok: management endpoints validate this metadata key as a list of route strings on write - "list[str]", (metadata or {}).get("denied_passthrough_routes") or [] - ) - for metadata in metadata_sources - ) - ) - if not denied_routes: + denied_sources: Final = [(metadata or {}).get("denied_passthrough_routes") for metadata in metadata_sources] + if not any(denied_sources): return None - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( - InitPassThroughEndpointHelpers, - ) - - forwarded_routes: Final = InitPassThroughEndpointHelpers.forwarded_routes(route) - return next( - ( - denied_route - for denied_route in denied_routes - if any( - RouteChecks._route_matches_denied_route(route=candidate, denied_route=denied_route) - for candidate in forwarded_routes - ) - ), - None, - ) + return auth_first_denied_route(list(RouteChecks.forwarded_pass_through_routes(route)), denied_sources) @staticmethod def passthrough_route_denied_exception(route: str, denied_route: str) -> HTTPException: @@ -881,11 +828,7 @@ class RouteChecks: a named route group like ``openai_routes`` is only ever compared as a path, and an entry that names no path segment (``*``, ``/*``) is skipped. """ - return any( - RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route) - for allowed_route in team_allowed_routes - if allowed_route.rstrip("*").strip("/") - ) + return auth_jwt_team_routes_grant(route, team_allowed_routes) @staticmethod def _jwt_team_allowed_routes(valid_token: UserAPIKeyAuth) -> Collection[str]: @@ -917,33 +860,13 @@ class RouteChecks: @staticmethod def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool: - """ - Check if route is a passthrough route. - Supports both exact match and prefix match. - """ - metadata: Final = user_api_key_dict.metadata - team_metadata: Final = user_api_key_dict.team_metadata or {} - if metadata is None and team_metadata is None: - return False - if "allowed_passthrough_routes" not in metadata and "allowed_passthrough_routes" not in team_metadata: - return False - if ( - metadata.get("allowed_passthrough_routes") is None - and team_metadata.get("allowed_passthrough_routes") is None - ): - return False - - allowed_passthrough_routes: Final = ( - metadata.get("allowed_passthrough_routes") or team_metadata.get("allowed_passthrough_routes") or [] + """The key's ``allowed_passthrough_routes`` when it lists any, otherwise the team's, matched by prefix.""" + return auth_allowed_routes_grant( + route, + key=_allowed_passthrough_routes(user_api_key_dict.metadata), + team=_allowed_passthrough_routes(user_api_key_dict.team_metadata), ) - # Check if route matches any allowed passthrough route (exact or prefix match) - for allowed_route in allowed_passthrough_routes: - if RouteChecks._route_matches_allowed_route(route=route, allowed_route=allowed_route): - return True - - return False - @staticmethod def is_assistants_api_request(request: Request) -> bool: """ diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index c72f58338c0..513cd84524f 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -3854,19 +3854,33 @@ dependencies = [ name = "litellm-config" version = "0.1.0" dependencies = [ - "litellm-error", + "litellm-python-compat", "litellm-router-types", "litellm-secret-value", + "litellm-starlette", "rstest", "serde", "serde_json", + "serde_with", "serde_yaml_ng", "strum", - "tempfile", "thiserror 2.0.19", "veil", ] +[[package]] +name = "litellm-config-load" +version = "0.1.0" +dependencies = [ + "litellm-config", + "litellm-error", + "rstest", + "serde", + "serde_yaml_ng", + "tempfile", + "thiserror 2.0.19", +] + [[package]] name = "litellm-core-utils" version = "0.1.0" @@ -3979,6 +3993,7 @@ dependencies = [ "http-body-util", "litellm-circuit-breaker", "litellm-config", + "litellm-config-load", "litellm-core-utils", "litellm-error", "litellm-gateway-auth", @@ -4036,6 +4051,18 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-gateway-auth-core" +version = "0.1.0" +dependencies = [ + "http 1.4.2", + "litellm-config", + "litellm-starlette", + "rstest", + "strum", + "thiserror 2.0.19", +] + [[package]] name = "litellm-gateway-forward" version = "0.1.0" @@ -4565,6 +4592,7 @@ dependencies = [ "criterion", "fancy-regex 0.19.2", "futures-util", + "http 1.4.2", "indexmap 2.14.0", "litellm-auth", "litellm-auth-aws", @@ -4577,9 +4605,11 @@ dependencies = [ "litellm-callbacks-python", "litellm-callbacks-s3-log", "litellm-circuit-breaker", + "litellm-config", "litellm-core-utils", "litellm-db", "litellm-error", + "litellm-gateway-auth-core", "litellm-host", "litellm-host-python", "litellm-http", @@ -4609,6 +4639,7 @@ dependencies = [ "litellm-storage-clickhouse", "litellm-token-counter", "litellm-tracing", + "parking_lot", "prost", "pyo3", "pyo3-async-runtimes", @@ -4641,6 +4672,7 @@ dependencies = [ "serde", "serde-pickle", "serde_json", + "serde_with", "thiserror 2.0.19", ] @@ -5026,6 +5058,15 @@ dependencies = [ "wiremock", ] +[[package]] +name = "litellm-starlette" +version = "0.1.0" +dependencies = [ + "rstest", + "serde", + "strum", +] + [[package]] name = "litellm-storage-clickhouse" version = "0.1.0" diff --git a/litellm/rust/Cargo.toml b/litellm/rust/Cargo.toml index 9a3c996c86e..ab8d6e5724d 100644 --- a/litellm/rust/Cargo.toml +++ b/litellm/rust/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] litellm-config = { path = "crates/config" } +litellm-config-load = { path = "crates/config-load" } litellm-error = { path = "crates/error" } litellm-router = { path = "crates/router" } litellm-router-types = { path = "crates/router-types" } @@ -29,6 +30,7 @@ litellm-gateway = { path = "crates/gateway" } litellm-gateway-forward = { path = "crates/gateway-forward" } litellm-gateway-inference = { path = "crates/gateway-inference" } litellm-gateway-auth = { path = "crates/gateway-auth" } +litellm-gateway-auth-core = { path = "crates/gateway-auth-core" } litellm-gateway-management = { path = "crates/gateway-management" } litellm-gateway-ui = { path = "crates/gateway-ui" } litellm-coroutine = { path = "crates/coroutine" } @@ -42,6 +44,7 @@ litellm-callbacks-budgets = { path = "crates/callbacks-budgets" } litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" } litellm-framer = { path = "crates/framer" } litellm-fnmatch = { path = "crates/fnmatch" } +litellm-starlette = { path = "crates/starlette" } litellm-auth = { path = "crates/auth" } litellm-auth-types = { path = "crates/auth-types" } litellm-secret-value = { path = "crates/secret-value" } diff --git a/litellm/rust/crates/config-load/AGENTS.md b/litellm/rust/crates/config-load/AGENTS.md new file mode 100644 index 00000000000..e10bee777df --- /dev/null +++ b/litellm/rust/crates/config-load/AGENTS.md @@ -0,0 +1,4 @@ +# Config load + +- This crate reads config files: the root file and every file its `include` names, merged breadth first, each file once. The typed `Config` lives in `litellm-config`, which reads no file and gets the merged document through `Config::from_value` +- A data crate never depends on this one. Only a host that starts from a path on disk (the gateway) does diff --git a/litellm/rust/crates/config-load/Cargo.toml b/litellm/rust/crates/config-load/Cargo.toml new file mode 100644 index 00000000000..ce00653e1e4 --- /dev/null +++ b/litellm/rust/crates/config-load/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "litellm-config-load" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Reads a LiteLLM config file and the files its `include` names into one typed Config" + +[dependencies] +litellm-config.workspace = true +serde.workspace = true +serde_yaml_ng = "0.10.0" +thiserror.workspace = true + +[dev-dependencies] +litellm-error.workspace = true +rstest.workspace = true +tempfile.workspace = true diff --git a/litellm/rust/crates/config-load/src/error.rs b/litellm/rust/crates/config-load/src/error.rs new file mode 100644 index 00000000000..3f2aef92294 --- /dev/null +++ b/litellm/rust/crates/config-load/src/error.rs @@ -0,0 +1,14 @@ +use std::{io, path::PathBuf}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("could not read config {}", path.display())] + Read { path: PathBuf, source: io::Error }, + #[error("invalid YAML config {}", path.display())] + ParseFile { + path: PathBuf, + source: serde_yaml_ng::Error, + }, + #[error(transparent)] + Config(#[from] litellm_config::Error), +} diff --git a/litellm/rust/crates/config/src/includes.rs b/litellm/rust/crates/config-load/src/includes.rs similarity index 100% rename from litellm/rust/crates/config/src/includes.rs rename to litellm/rust/crates/config-load/src/includes.rs diff --git a/litellm/rust/crates/config-load/src/lib.rs b/litellm/rust/crates/config-load/src/lib.rs new file mode 100644 index 00000000000..6dd8779ac46 --- /dev/null +++ b/litellm/rust/crates/config-load/src/lib.rs @@ -0,0 +1,13 @@ +mod error; +mod includes; + +use std::path::Path; + +use litellm_config::Config; + +pub use error::Error; + +/// The config at `path` with every file its `include` names merged in. +pub fn load(path: impl AsRef) -> Result { + Ok(Config::from_value(includes::load(path.as_ref())?)?) +} diff --git a/litellm/rust/crates/config-load/tests/load.rs b/litellm/rust/crates/config-load/tests/load.rs new file mode 100644 index 00000000000..d7e687ccd9f --- /dev/null +++ b/litellm/rust/crates/config-load/tests/load.rs @@ -0,0 +1,106 @@ +use litellm_config_load::Error; +use rstest::{fixture, rstest}; +use tempfile::TempDir; + +#[fixture] +fn directory() -> TempDir { + tempfile::tempdir().unwrap() +} + +#[fixture] +fn model_list_yaml() -> &'static str { + r#" +model_list: + - model_name: assistant + litellm_params: + model: anthropic/test-model + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: local + litellm_params: + model: test-model + api_base: http://localhost:8000/v1 + custom_llm_provider: openai +"# +} + +#[rstest] +fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) { + let path = directory.path().join("config.yaml"); + std::fs::write(&path, model_list_yaml).unwrap(); + + let config = litellm_config_load::load(path).unwrap(); + assert_eq!(config.model_list.len(), 2); + let anthropic = &config.model_list[0]; + assert_eq!(anthropic.model_name, "assistant"); + assert_eq!(anthropic.litellm_params.model, "anthropic/test-model"); + assert_eq!( + anthropic.litellm_params.api_key.as_ref().unwrap().expose(), + "os.environ/ANTHROPIC_API_KEY" + ); + assert!(anthropic.litellm_params.api_base.is_none()); + assert!(anthropic.litellm_params.custom_llm_provider.is_none()); + let local = &config.model_list[1]; + assert_eq!(local.model_name, "local"); + assert_eq!(local.litellm_params.model, "test-model"); + assert!(local.litellm_params.api_key.is_none()); + assert_eq!( + local.litellm_params.api_base.as_deref(), + Some("http://localhost:8000/v1") + ); + assert_eq!( + local.litellm_params.custom_llm_provider.as_deref(), + Some("openai") + ); +} + +#[rstest] +fn distinguishes_read_errors_from_parse_errors(directory: TempDir) { + let error = litellm_config_load::load(directory.path().join("missing.yaml")).unwrap_err(); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("missing.yaml") + ); + assert!(matches!( + error, + Error::Read { source, .. } if source.kind() == std::io::ErrorKind::NotFound + )); +} + +#[rstest] +#[case::policy_pipeline("../../../proxy/example_config_yaml/test_pipeline_config.yaml")] +#[case::gateway("../../../../tests/e2e/gateway/stage_mirror_ci_config.yml")] +#[case::pass_through("../../../proxy/example_config_yaml/pass_through_config.yaml")] +fn parses_representative_python_configs(#[case] relative_path: &str) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../config") + .join(relative_path); + let config = litellm_config_load::load(path).unwrap(); + assert!(!config.model_list.is_empty()); +} + +#[rstest] +fn resolves_nested_includes_once_in_breadth_first_order() { + let directory = TempDir::new().unwrap(); + let root = directory.path(); + std::fs::create_dir(root.join("nested")).unwrap(); + std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap(); + std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap(); + std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap(); + std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap(); + + let config = litellm_config_load::load(root.join("config.yaml")).unwrap(); + assert_eq!( + config + .model_list + .iter() + .map(|model| model.model_name.as_str()) + .collect::>(), + ["root", "first", "second", "third"] + ); + assert_eq!( + config.general_settings.master_key.unwrap().expose(), + "third" + ); + assert!(config.include.is_empty()); +} diff --git a/litellm/rust/crates/config/AGENTS.md b/litellm/rust/crates/config/AGENTS.md index 9f6f1db9a67..e4c991694fa 100644 --- a/litellm/rust/crates/config/AGENTS.md +++ b/litellm/rust/crates/config/AGENTS.md @@ -1,3 +1,11 @@ +## Scope + +- This crate is the typed data model of a config and reads no file. Reading files and `include` live in `config-load`, which hands the merged document to `Config::from_value` + ## Redis settings - This crate only deserializes YAML. `RedisSettings` is the raw `coordination_redis` or `cache_params` map and `CacheSettings` tells a `type: redis` cache apart from the rest. Reading those maps the way `litellm/_redis.py` does belongs to `gateway/src/redis`, and turning merged kwargs into a `Resolution` belongs to `redis-resolve` + +## Route settings + +- `RouteList` is the typed form of a route list in config and key metadata, and both hosts read through it. A value that is not a list of strings is `RouteList::Malformed`, which grants nothing. A route template inside such a list is starlette syntax, so it is parsed by `litellm-starlette`, never here diff --git a/litellm/rust/crates/config/Cargo.toml b/litellm/rust/crates/config/Cargo.toml index a433aacc1c9..c8fe161664b 100644 --- a/litellm/rust/crates/config/Cargo.toml +++ b/litellm/rust/crates/config/Cargo.toml @@ -6,16 +6,17 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-python-compat.workspace = true litellm-router-types.workspace = true litellm-secret-value.workspace = true +litellm-starlette.workspace = true serde.workspace = true +serde_with.workspace = true serde_yaml_ng = "0.10.0" strum.workspace = true thiserror.workspace = true veil.workspace = true [dev-dependencies] -litellm-error.workspace = true rstest.workspace = true serde_json.workspace = true -tempfile.workspace = true diff --git a/litellm/rust/crates/config/src/error.rs b/litellm/rust/crates/config/src/error.rs index 300f15380bd..e3db06bfdd2 100644 --- a/litellm/rust/crates/config/src/error.rs +++ b/litellm/rust/crates/config/src/error.rs @@ -1,14 +1,5 @@ -use std::{io, path::PathBuf}; - #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("could not read config {}", path.display())] - Read { path: PathBuf, source: io::Error }, #[error("invalid YAML config")] Parse(#[from] serde_yaml_ng::Error), - #[error("invalid YAML config {}", path.display())] - ParseFile { - path: PathBuf, - source: serde_yaml_ng::Error, - }, } diff --git a/litellm/rust/crates/config/src/lib.rs b/litellm/rust/crates/config/src/lib.rs index 46657aa91e7..fc7b81f9192 100644 --- a/litellm/rust/crates/config/src/lib.rs +++ b/litellm/rust/crates/config/src/lib.rs @@ -1,27 +1,28 @@ mod error; -mod includes; mod mcp; mod model; +mod pass_through; mod redis; +mod routes; mod settings; mod value; -use std::{fmt, path::Path}; - use serde::Deserialize; pub use error::Error; pub use litellm_router_types::{LitellmParams, Spelled}; pub use mcp::{McpAuth, McpServer, McpTransport}; pub use model::Model; +pub use pass_through::{PassThroughEndpoint, PassThroughGuardrail}; pub use redis::{CacheSettings, RedisSettings}; +pub use routes::RouteList; pub use settings::{ ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings, TracingStoreSettings, }; pub use value::{AdditionalFields, Object, Value}; -#[derive(Clone, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] #[serde(default)] pub struct Config { pub model_list: Box<[Model]>, @@ -52,45 +53,12 @@ pub struct Config { pub additional_fields: AdditionalFields, } -impl fmt::Debug for Config { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Config") - .field("model_list", &self.model_list) - .field("general_settings", &self.general_settings) - .field("router_settings", &self.router_settings) - .field("litellm_settings", &self.litellm_settings) - .field("environment_variables", &self.environment_variables) - .field("callback_settings", &self.callback_settings) - .field("assistant_settings", &self.assistant_settings) - .field("default_vertex_config", &self.default_vertex_config) - .field("mcp_servers", &self.mcp_servers) - .field("credential_list", &self.credential_list) - .field("guardrails", &self.guardrails) - .field("prompts", &self.prompts) - .field("sandbox_tools", &self.sandbox_tools) - .field("search_tools", &self.search_tools) - .field("files_settings", &self.files_settings) - .field("finetune_settings", &self.finetune_settings) - .field("mcp_tools", &self.mcp_tools) - .field("vector_store_registry", &self.vector_store_registry) - .field("worker_registry", &self.worker_registry) - .field("agents", &self.agents) - .field("agent_list", &self.agent_list) - .field("policies", &self.policies) - .field("policy_attachments", &self.policy_attachments) - .field("include", &self.include) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} - impl Config { pub fn from_yaml(yaml: &str) -> Result { Ok(serde_yaml_ng::from_str(yaml)?) } - pub fn load(path: impl AsRef) -> Result { - Ok(serde_yaml_ng::from_value(includes::load(path.as_ref())?)?) + pub fn from_value(document: Value) -> Result { + Ok(serde_yaml_ng::from_value(document)?) } } diff --git a/litellm/rust/crates/config/src/mcp.rs b/litellm/rust/crates/config/src/mcp.rs index 00480fa5be1..56d9f75d2e2 100644 --- a/litellm/rust/crates/config/src/mcp.rs +++ b/litellm/rust/crates/config/src/mcp.rs @@ -1,11 +1,13 @@ -use std::{collections::BTreeMap, fmt}; +use std::collections::BTreeMap; use litellm_secret_value::SecretValue; use serde::Deserialize; +use veil::Redact; use crate::Object; -#[derive(Clone, Default, Deserialize)] +/// `command` and `args` can carry credentials, and the `SecretValue`s redact themselves. +#[derive(Clone, Default, Deserialize, Redact)] #[serde(default)] pub struct McpServer { pub server_id: Option, @@ -14,7 +16,9 @@ pub struct McpServer { pub mcp_info: Object, pub transport: McpTransport, pub url: Option, + #[redact(fixed = 3)] pub command: Option, + #[redact(fixed = 3)] pub args: Box<[String]>, pub env: BTreeMap, pub auth_type: Option, @@ -29,18 +33,6 @@ pub struct McpServer { pub unsupported: Object, } -impl fmt::Debug for McpServer { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("McpServer") - .field("transport", &self.transport) - .field("auth_type", &self.auth_type) - .field("timeout", &self.timeout) - .field("max_concurrent_requests", &self.max_concurrent_requests) - .finish_non_exhaustive() - } -} - #[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)] #[serde(rename_all = "snake_case")] pub enum McpTransport { diff --git a/litellm/rust/crates/config/src/model.rs b/litellm/rust/crates/config/src/model.rs index c0f8516103d..f5040744fec 100644 --- a/litellm/rust/crates/config/src/model.rs +++ b/litellm/rust/crates/config/src/model.rs @@ -1,11 +1,9 @@ -use std::fmt; - use litellm_router_types::LitellmParams; use serde::Deserialize; use crate::{AdditionalFields, Object}; -#[derive(Clone, Deserialize)] +#[derive(Clone, Debug, Deserialize)] pub struct Model { pub model_name: String, pub litellm_params: LitellmParams, @@ -15,16 +13,3 @@ pub struct Model { #[serde(flatten)] pub additional_fields: AdditionalFields, } - -impl fmt::Debug for Model { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Model") - .field("model_name", &self.model_name) - .field("litellm_params", &self.litellm_params) - .field("model_info", &self.model_info) - .field("blocked", &self.blocked) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} diff --git a/litellm/rust/crates/config/src/pass_through.rs b/litellm/rust/crates/config/src/pass_through.rs new file mode 100644 index 00000000000..03fb4f2547b --- /dev/null +++ b/litellm/rust/crates/config/src/pass_through.rs @@ -0,0 +1,61 @@ +use std::collections::BTreeMap; + +use litellm_python_compat::lenient::{IsTrue, Truthy}; +use litellm_secret_value::SecretValue; +use litellm_starlette::RouteTemplate; +use serde::Deserialize; +use serde_with::serde_as; + +use crate::{Object, Spelled}; + +/// A `general_settings.pass_through_endpoints` entry: a proxy `path` forwarded to `target`. Each +/// field is read the way `initialize_pass_through_endpoints` reads the config dict. +#[serde_as] +#[derive(Clone, Debug, Deserialize)] +pub struct PassThroughEndpoint { + pub id: Option, + pub path: RouteTemplate, + /// Registration skips an endpoint without one. + pub target: Option, + /// Upstream credentials and `os.environ/` references, so only the keys are printed. + #[serde(default)] + pub headers: Object, + #[serde(default)] + pub default_query_params: Object, + /// `endpoint.get("include_subpath", False) is True` + #[serde_as(as = "IsTrue")] + #[serde(default)] + pub include_subpath: bool, + pub cost_per_request: Option>, + pub timeout: Option>, + /// `endpoint.get("auth", True) is True`, which decides whether a request needs a key. + /// Registration still reads `str(auth).lower() == "true"`, until both share one rule. + #[serde_as(as = "IsTrue")] + #[serde(default = "auth_default")] + pub auth: bool, + pub methods: Option>, + pub guardrails: Option>>, + #[serde_as(as = "Truthy")] + #[serde(default)] + pub forward_headers: bool, + #[serde_as(as = "Truthy")] + #[serde(default)] + pub merge_query_params: bool, + /// `"langfuse"` reads the key out of a Basic credential. + pub custom_auth_parser: Option, + #[serde(flatten)] + pub additional_fields: Object, +} + +const fn auth_default() -> bool { + true +} + +/// `PassThroughGuardrailSettings`: which request and response fields one guardrail reads. +#[derive(Clone, Debug, Deserialize)] +pub struct PassThroughGuardrail { + pub request_fields: Option>, + pub response_fields: Option>, + #[serde(flatten)] + pub additional_fields: Object, +} diff --git a/litellm/rust/crates/config/src/routes.rs b/litellm/rust/crates/config/src/routes.rs new file mode 100644 index 00000000000..11e55cca6e8 --- /dev/null +++ b/litellm/rust/crates/config/src/routes.rs @@ -0,0 +1,6 @@ +use litellm_python_compat::lenient::Lenient; + +/// A route list from config or key metadata, such as `public_routes` or +/// `allowed_passthrough_routes`. A value that is not a list of strings is `Malformed`, and each +/// reader decides what that grants: an allow list grants nothing, a deny list denies every route. +pub type RouteList = Lenient>; diff --git a/litellm/rust/crates/config/src/settings.rs b/litellm/rust/crates/config/src/settings.rs index 1bfc19983f3..cd4bc5e5bc7 100644 --- a/litellm/rust/crates/config/src/settings.rs +++ b/litellm/rust/crates/config/src/settings.rs @@ -1,10 +1,10 @@ -use std::fmt; - use litellm_secret_value::SecretValue; use serde::Deserialize; +use veil::Redact; use crate::{ - AdditionalFields, CacheSettings, Object, RedisSettings, Spelled, Value, value::one_or_many, + AdditionalFields, CacheSettings, Object, PassThroughEndpoint, RedisSettings, Spelled, Value, + value::one_or_many, }; #[derive(Clone, Debug, Deserialize)] @@ -13,7 +13,7 @@ pub enum TracingStoreKind { Clickhouse, } -#[derive(Clone, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct ClickHouseStoreSettings { #[serde(rename = "type")] @@ -23,17 +23,6 @@ pub struct ClickHouseStoreSettings { pub retention_days: Option>, } -impl fmt::Debug for ClickHouseStoreSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ClickHouseStoreSettings") - .field("kind", &self.kind) - .field("database", &self.database) - .field("retention_days", &self.retention_days) - .finish() - } -} - #[derive(Clone, Debug, Deserialize)] #[serde(untagged)] pub enum TracingStoreSettings { @@ -48,7 +37,7 @@ pub struct TracingSettings { pub additional_fields: AdditionalFields, } -#[derive(Clone, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(default)] pub struct GeneralSettings { pub completion_model: Option, @@ -81,6 +70,7 @@ pub struct GeneralSettings { pub coordination_redis: Option, pub mcp_allowed_hosts: Option>, pub mcp_allowed_origins: Box<[String]>, + pub pass_through_endpoints: Option>, #[serde(flatten)] pub additional_fields: AdditionalFields, } @@ -118,38 +108,13 @@ impl Default for GeneralSettings { coordination_redis: None, mcp_allowed_hosts: None, mcp_allowed_origins: Box::default(), - additional_fields: AdditionalFields::new(), + pass_through_endpoints: None, + additional_fields: AdditionalFields::default(), } } } -impl fmt::Debug for GeneralSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("GeneralSettings") - .field("completion_model", &self.completion_model) - .field( - "max_in_flight_requests_per_worker", - &self.max_in_flight_requests_per_worker, - ) - .field( - "max_queued_requests_per_worker", - &self.max_queued_requests_per_worker, - ) - .field( - "admission_queue_timeout_seconds", - &self.admission_queue_timeout_seconds, - ) - .field("master_key", &self.master_key) - .field("database_url", &self.database_url) - .field("tracing", &self.tracing) - .field("store_model_in_db", &self.store_model_in_db) - .field("additional_fields", &self.additional_fields.keys()) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] #[serde(default)] pub struct RouterSettings { pub routing_strategy: Option, @@ -175,38 +140,8 @@ pub struct RouterSettings { pub additional_fields: AdditionalFields, } -impl fmt::Debug for RouterSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RouterSettings") - .field("routing_strategy", &self.routing_strategy) - .field("routing_strategy_args", &self.routing_strategy_args) - .field("routing_groups", &self.routing_groups) - .field("retry_policy", &self.retry_policy) - .field("model_group_retry_policy", &self.model_group_retry_policy) - .field( - "model_group_affinity_config", - &self.model_group_affinity_config, - ) - .field("allowed_fails", &self.allowed_fails) - .field("cooldown_time", &self.cooldown_time) - .field("num_retries", &self.num_retries) - .field("timeout", &self.timeout) - .field("max_retries", &self.max_retries) - .field("retry_after", &self.retry_after) - .field("fallbacks", &self.fallbacks) - .field("context_window_fallbacks", &self.context_window_fallbacks) - .field("model_group_alias", &self.model_group_alias) - .field("enable_tag_filtering", &self.enable_tag_filtering) - .field("weights", &self.weights) - .field("tag_routing_prefix", &self.tag_routing_prefix) - .field("optional_pre_call_checks", &self.optional_pre_call_checks) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} - -#[derive(Clone, Default, Deserialize)] +/// Callback entries can carry credentials inline, so only whether they are set is printed. +#[derive(Clone, Default, Deserialize, Redact)] #[serde(default)] pub struct LiteLlmSettings { pub ssl_verify: Option>, @@ -224,47 +159,16 @@ pub struct LiteLlmSettings { pub cache: Option, pub cache_params: Option, #[serde(deserialize_with = "one_or_many")] + #[redact(fixed = 3)] pub callbacks: Option>, #[serde(deserialize_with = "one_or_many")] + #[redact(fixed = 3)] pub success_callback: Option>, #[serde(deserialize_with = "one_or_many")] + #[redact(fixed = 3)] pub failure_callback: Option>, pub json_logs: Option, pub set_verbose: Option, #[serde(flatten)] pub additional_fields: AdditionalFields, } - -impl fmt::Debug for LiteLlmSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LiteLlmSettings") - .field("drop_params", &self.drop_params) - .field("request_timeout", &self.request_timeout) - .field("num_retries", &self.num_retries) - .field("cache", &self.cache) - .field("cache_params", &self.cache_params) - .field( - "callbacks", - &self.callbacks.as_ref().map(|callbacks| callbacks.len()), - ) - .field( - "success_callback", - &self - .success_callback - .as_ref() - .map(|callbacks| callbacks.len()), - ) - .field( - "failure_callback", - &self - .failure_callback - .as_ref() - .map(|callbacks| callbacks.len()), - ) - .field("json_logs", &self.json_logs) - .field("set_verbose", &self.set_verbose) - .field("additional_fields", &self.additional_fields.keys()) - .finish() - } -} diff --git a/litellm/rust/crates/config/src/value.rs b/litellm/rust/crates/config/src/value.rs index 667a5afa130..a86d93cdfa7 100644 --- a/litellm/rust/crates/config/src/value.rs +++ b/litellm/rust/crates/config/src/value.rs @@ -3,7 +3,8 @@ use std::{collections::BTreeMap, fmt, ops::Deref}; use serde::{Deserialize, Deserializer}; pub type Value = serde_yaml_ng::Value; -pub type AdditionalFields = BTreeMap; +/// The keys a config section has beyond its typed fields, printed as keys only. +pub type AdditionalFields = Object; #[derive(Clone, Default, Deserialize)] #[serde(transparent)] diff --git a/litellm/rust/crates/config/tests/config.rs b/litellm/rust/crates/config/tests/config.rs index ae0930b6bde..5c5ddb3568b 100644 --- a/litellm/rust/crates/config/tests/config.rs +++ b/litellm/rust/crates/config/tests/config.rs @@ -1,57 +1,5 @@ use litellm_config::{Config, Error, Spelled, TracingStoreSettings}; -use rstest::{fixture, rstest}; -use tempfile::TempDir; - -#[fixture] -fn directory() -> TempDir { - tempfile::tempdir().unwrap() -} - -#[fixture] -fn model_list_yaml() -> &'static str { - r#" -model_list: - - model_name: assistant - litellm_params: - model: anthropic/test-model - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: local - litellm_params: - model: test-model - api_base: http://localhost:8000/v1 - custom_llm_provider: openai -"# -} - -#[rstest] -fn loads_model_list_from_file(directory: TempDir, model_list_yaml: &str) { - let path = directory.path().join("config.yaml"); - std::fs::write(&path, model_list_yaml).unwrap(); - - let config = Config::load(path).unwrap(); - assert_eq!(config.model_list.len(), 2); - let anthropic = &config.model_list[0]; - assert_eq!(anthropic.model_name, "assistant"); - assert_eq!(anthropic.litellm_params.model, "anthropic/test-model"); - assert_eq!( - anthropic.litellm_params.api_key.as_ref().unwrap().expose(), - "os.environ/ANTHROPIC_API_KEY" - ); - assert!(anthropic.litellm_params.api_base.is_none()); - assert!(anthropic.litellm_params.custom_llm_provider.is_none()); - let local = &config.model_list[1]; - assert_eq!(local.model_name, "local"); - assert_eq!(local.litellm_params.model, "test-model"); - assert!(local.litellm_params.api_key.is_none()); - assert_eq!( - local.litellm_params.api_base.as_deref(), - Some("http://localhost:8000/v1") - ); - assert_eq!( - local.litellm_params.custom_llm_provider.as_deref(), - Some("openai") - ); -} +use rstest::rstest; #[rstest] fn config_debug_redacts_api_keys() { @@ -82,20 +30,6 @@ fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) { assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_)))); } -#[rstest] -fn distinguishes_read_errors_from_parse_errors(directory: TempDir) { - let error = Config::load(directory.path().join("missing.yaml")).unwrap_err(); - assert!( - litellm_error::Report::new(&error) - .to_string() - .contains("missing.yaml") - ); - assert!(matches!( - error, - Error::Read { source, .. } if source.kind() == std::io::ErrorKind::NotFound - )); -} - #[rstest] #[case::literal("secret-master-key")] #[case::reference("os.environ/LITELLM_MASTER_KEY")] @@ -334,15 +268,6 @@ include: assert_eq!(&*config.include, &["models.yaml"]); } -#[rstest] -#[case::policy_pipeline("../../../proxy/example_config_yaml/test_pipeline_config.yaml")] -#[case::gateway("../../../../tests/e2e/gateway/stage_mirror_ci_config.yml")] -fn parses_representative_python_configs(#[case] relative_path: &str) { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative_path); - let config = Config::load(path).unwrap(); - assert!(!config.model_list.is_empty()); -} - #[rstest] #[case::one("callbacks: custom_callbacks.logger", 1)] #[case::many("callbacks: [prometheus, otel]", 2)] @@ -387,32 +312,6 @@ fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secre assert!(!format!("{config:?}").contains(secret)); } -#[rstest] -fn resolves_nested_includes_once_in_breadth_first_order() { - let directory = TempDir::new().unwrap(); - let root = directory.path(); - std::fs::create_dir(root.join("nested")).unwrap(); - std::fs::write(root.join("config.yaml"), "include: [nested/first.yaml, second.yaml]\nmodel_list: [{model_name: root, litellm_params: {model: root}}]\n").unwrap(); - std::fs::write(root.join("nested/first.yaml"), "include: [third.yaml]\nmodel_list: [{model_name: first, litellm_params: {model: first}}]\n").unwrap(); - std::fs::write(root.join("second.yaml"), "general_settings: {master_key: second}\nmodel_list: [{model_name: second, litellm_params: {model: second}}]\n").unwrap(); - std::fs::write(root.join("nested/third.yaml"), "include: [../config.yaml]\ngeneral_settings: {master_key: third}\nmodel_list: [{model_name: third, litellm_params: {model: third}}]\n").unwrap(); - - let config = Config::load(root.join("config.yaml")).unwrap(); - assert_eq!( - config - .model_list - .iter() - .map(|model| model.model_name.as_str()) - .collect::>(), - ["root", "first", "second", "third"] - ); - assert_eq!( - config.general_settings.master_key.unwrap().expose(), - "third" - ); - assert!(config.include.is_empty()); -} - #[rstest] fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() { let config = Config::from_yaml("mcp_servers:\n docs:\n url: https://example.test/private-secret/mcp\n authentication_token: upstream-secret\n static_headers: {x-token: header-secret}\n env: {TOKEN: env-secret}\n args: [argument-secret]\n client_secret: oauth-secret\n allowed_tools: [search]\n").unwrap(); @@ -466,3 +365,115 @@ fn non_redis_cache_settings_remain_opaque() { Some(litellm_config::CacheSettings::Other(_)) )); } + +fn pass_through(entry: &str) -> litellm_config::PassThroughEndpoint { + let config = Config::from_yaml(&format!( + "general_settings: {{pass_through_endpoints: [{entry}]}}" + )) + .unwrap(); + config.general_settings.pass_through_endpoints.unwrap()[0].clone() +} + +#[rstest] +fn a_pass_through_endpoint_reads_its_python_fields() { + let endpoint = pass_through( + "{path: '/bria/{item_id:int}', target: https://upstream.example/v1, include_subpath: true, \ + headers: {Authorization: os.environ/BRIA_KEY}, cost_per_request: '0.5', methods: [POST], \ + guardrails: {pii: {request_fields: [text]}, toxicity: null}, custom_auth_parser: langfuse, display_name: Bria}", + ); + assert_eq!( + endpoint.path, + litellm_starlette::RouteTemplate::parse("/bria/{item_id:int}") + ); + assert_eq!( + endpoint.target.unwrap().expose(), + "https://upstream.example/v1" + ); + assert!(endpoint.include_subpath); + assert_eq!( + endpoint.headers.keys().collect::>(), + ["Authorization"] + ); + assert_eq!(endpoint.methods.as_deref(), Some(&["POST".to_owned()][..])); + let guardrails = endpoint.guardrails.unwrap(); + assert_eq!( + guardrails["pii"] + .as_ref() + .unwrap() + .request_fields + .as_deref(), + Some(&["text".to_owned()][..]) + ); + assert!(guardrails["toxicity"].is_none()); + assert_eq!(endpoint.custom_auth_parser.as_deref(), Some("langfuse")); + assert!(endpoint.additional_fields.contains_key("display_name")); +} + +/// `endpoint.get("auth", True) is True`, `endpoint.get("include_subpath", False) is True` and +/// `bool(forward_headers)`, as the proxy reads them. +#[rstest] +#[case::nothing_set("", true, false, false)] +#[case::booleans( + "auth: false, include_subpath: true, forward_headers: true", + false, + true, + true +)] +#[case::strings( + "auth: 'true', include_subpath: 'true', forward_headers: 'false'", + false, + false, + true +)] +#[case::numbers("auth: 1, include_subpath: 1, forward_headers: 0", false, false, false)] +fn pass_through_flags_read_like_python( + #[case] flags: &str, + #[case] auth: bool, + #[case] include_subpath: bool, + #[case] forward_headers: bool, +) { + let separator = if flags.is_empty() { "" } else { ", " }; + let endpoint = pass_through(&format!( + "{{path: /x, target: https://t.example{separator}{flags}}}" + )); + assert_eq!( + ( + endpoint.auth, + endpoint.include_subpath, + endpoint.forward_headers + ), + (auth, include_subpath, forward_headers) + ); +} + +#[rstest] +fn a_pass_through_endpoint_without_a_path_fails_the_load_like_registration() { + assert!( + Config::from_yaml( + "general_settings: {pass_through_endpoints: [{target: https://t.example}]}" + ) + .is_err() + ); +} + +#[rstest] +fn pass_through_debug_hides_the_target_and_header_values() { + let config = Config::from_yaml( + "general_settings: {pass_through_endpoints: [{path: /x, target: https://user:target-secret@t.example, headers: {Authorization: header-secret}}]}", + ) + .unwrap(); + let debug = format!("{config:?}"); + assert!(!debug.contains("target-secret") && !debug.contains("header-secret")); +} + +#[rstest] +#[case::callbacks("callbacks")] +#[case::success_callback("success_callback")] +#[case::failure_callback("failure_callback")] +fn debug_hides_inline_callback_settings(#[case] field: &str) { + let config = Config::from_yaml(&format!( + "litellm_settings: {{{field}: [{{token: callback-list-secret}}]}}" + )) + .unwrap(); + assert!(!format!("{config:?}").contains("callback-list-secret")); +} diff --git a/litellm/rust/crates/config/tests/routes.rs b/litellm/rust/crates/config/tests/routes.rs new file mode 100644 index 00000000000..329ee9cf6cf --- /dev/null +++ b/litellm/rust/crates/config/tests/routes.rs @@ -0,0 +1,17 @@ +use litellm_config::RouteList; +use rstest::rstest; + +#[rstest] +#[case::list("[/a, /b/*]", RouteList::Valid(["/a".to_owned(), "/b/*".to_owned()].into()))] +#[case::empty("[]", RouteList::Valid([].into()))] +#[case::bare_string("/a/*", RouteList::Malformed)] +#[case::number("5", RouteList::Malformed)] +#[case::mapping("{a: 1}", RouteList::Malformed)] +#[case::non_string_entry("[/a, 5, /b/*]", RouteList::Malformed)] +#[case::null_entry("[/a, null]", RouteList::Malformed)] +fn route_lists_read(#[case] yaml: &str, #[case] expected: RouteList) { + assert_eq!( + serde_yaml_ng::from_str::(yaml).unwrap(), + expected + ); +} diff --git a/litellm/rust/crates/gateway-auth-core/Cargo.toml b/litellm/rust/crates/gateway-auth-core/Cargo.toml new file mode 100644 index 00000000000..1a8f933498b --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "litellm-gateway-auth-core" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +http.workspace = true +litellm-config.workspace = true +litellm-starlette.workspace = true +strum.workspace = true +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm/rust/crates/gateway-auth-core/src/lib.rs b/litellm/rust/crates/gateway-auth-core/src/lib.rs new file mode 100644 index 00000000000..4aeaecbb447 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/lib.rs @@ -0,0 +1,7 @@ +pub mod routes; +#[cfg(test)] +mod test_support; + +pub use routes::{ + ManagedRoute, MissingRouteGroup, RouteAllowlist, RouteGroup, RouteGroups, matches_wildcard, +}; diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/AGENTS.md b/litellm/rust/crates/gateway-auth-core/src/routes/AGENTS.md new file mode 100644 index 00000000000..6fa81126459 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/AGENTS.md @@ -0,0 +1,10 @@ +# Routes + +Whether a route may be called + +## `error.rs` + +- Every error a file here can produce lives in `routes/error.rs`. No other file declares its own error type +- `MissingRouteGroup` is a settings build failure from `RouteGroups::new`, never a request denial, so it stays out of `crate::Error` +- Every message is in `#[error(...)]`, built from the variant's typed fields. Keep each opening sentence stable, since clients match on it +- A file that only answers a question (`passthrough.rs`, `managed.rs`, `table.rs`) returns a plain value and adds nothing here diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/error.rs b/litellm/rust/crates/gateway-auth-core/src/routes/error.rs new file mode 100644 index 00000000000..bf58966e542 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/error.rs @@ -0,0 +1,8 @@ +use super::RouteGroup; + +/// A settings build failure, never a request denial. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("the host's route table has no {} group", <&str>::from(*group))] +pub struct MissingRouteGroup { + pub group: RouteGroup, +} diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/groups.rs b/litellm/rust/crates/gateway-auth-core/src/routes/groups.rs new file mode 100644 index 00000000000..03552a1c48e --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/groups.rs @@ -0,0 +1,224 @@ +use std::{collections::HashMap, sync::Arc}; + +use strum::VariantArray; + +use super::{MissingRouteGroup, table::RouteGroup, template::Template}; + +#[derive(Debug)] +pub(super) struct Entries { + pub(super) routes: Box<[String]>, + pub(super) templates: Box<[Template]>, +} + +impl Entries { + pub(super) fn matches(&self, route: &str) -> bool { + self.templates + .iter() + .any(|template| template.matches(route)) + } + + fn matches_compiled(&self, route: &str) -> bool { + self.templates + .iter() + .any(|template| template.matches_compiled(route)) + } +} + +/// `LiteLLMRoutes` by member name, which the host hands in with its settings. +#[derive(Clone, Debug, Default)] +pub struct RouteGroups(Arc>); + +/// An operator's route list paired with the groups its entries may name, so the list is only ever +/// read through `_allowed_routes_check`. +#[derive(Clone, Copy, Debug)] +pub struct RouteAllowlist<'a> { + entries: &'a [String], + groups: &'a RouteGroups, +} + +impl<'a> RouteAllowlist<'a> { + pub fn permits(&self, route: &str) -> bool { + self.groups.is_route_allowed(route, self.entries) + } + + pub fn entries(&self) -> &'a [String] { + self.entries + } + + /// `get_actual_routes` + pub fn expanded(&self) -> Vec { + self.groups.actual_routes(self.entries) + } + + /// `jwt_team_routes_grant_pass_through`: explicit paths and trailing-wildcard prefixes grant, + /// group names and blanket entries (`*`, `/*`) never do. + pub fn grants_pass_through(&self, route: &str) -> bool { + self.entries + .iter() + .filter(|allowed| !allowed.trim_end_matches('*').trim_matches('/').is_empty()) + .any(|allowed| matches_wildcard(route, allowed)) + } +} + +impl RouteGroups { + /// Fails on a table without a group the core's rules read, so a group the host renames stops + /// the settings build instead of reading as empty. + pub fn new( + groups: impl IntoIterator)>, + ) -> Result { + let groups: HashMap = groups + .into_iter() + .map(|(name, routes)| { + let templates = routes + .iter() + .map(|route| Template::compile(route)) + .collect(); + ( + name, + Entries { + routes: routes.into(), + templates, + }, + ) + }) + .collect(); + let missing = RouteGroup::VARIANTS + .iter() + .copied() + .find(|group| !groups.contains_key(<&str>::from(*group))); + match missing { + Some(group) => Err(MissingRouteGroup { group }), + None => Ok(Self(Arc::new(groups))), + } + } + + pub(super) fn entries(&self, group: RouteGroup) -> Option<&Entries> { + self.0.get(<&str>::from(group)) + } + + /// `LiteLLMRoutes..value` + pub fn routes(&self, group: RouteGroup) -> &[String] { + self.entries(group) + .map(|entries| &*entries.routes) + .unwrap_or_default() + } + + /// `route in LiteLLMRoutes..value`: the route is one of the group's entries as written. + pub fn lists(&self, route: &str, group: RouteGroup) -> bool { + self.routes(group).iter().any(|entry| entry == route) + } + + /// The route matches one of the group's entries read as starlette templates. + pub fn matches(&self, route: &str, group: RouteGroup) -> bool { + self.entries(group) + .is_some_and(|entries| entries.matches(route)) + } + + /// `_allowed_routes_check` over an operator's list: each entry is a `LiteLLMRoutes` group name + /// whose templates match as starlette `compile_path` compiles them, + /// an exact route, or a prefix ending in `*`. + pub fn is_route_allowed>(&self, route: &str, allowed_routes: &[S]) -> bool { + allowed_routes + .iter() + .any(|allowed| match self.0.get(allowed.as_ref()) { + Some(entries) => entries.matches_compiled(route), + None => matches_wildcard(route, allowed.as_ref()), + }) + } + + pub fn allowlist<'a>(&'a self, entries: &'a [String]) -> RouteAllowlist<'a> { + RouteAllowlist { + entries, + groups: self, + } + } + + /// `get_actual_routes`: group names expanded to their routes, anything else kept as given. + pub fn actual_routes>(&self, allowed_routes: &[S]) -> Vec { + allowed_routes + .iter() + .flat_map(|allowed| match self.0.get(allowed.as_ref()) { + Some(entries) => entries.routes.to_vec(), + None => vec![allowed.as_ref().to_owned()], + }) + .collect() + } +} + +/// `route_matches_wildcard_pattern`: a trailing `*` matches by prefix, anything else exactly. +pub fn matches_wildcard(route: &str, pattern: &str) -> bool { + match pattern.strip_suffix('*') { + Some(prefix) => route.starts_with(prefix), + None => route == pattern, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::test_support::{route_groups, route_table}; + + #[rstest] + #[case::exact("/scim/v2", "/scim/v2", true)] + #[case::wildcard_child("/scim/v2/Users/1", "/scim/v2/*", true)] + #[case::wildcard_other("/chat/completions", "/scim/v2/*", false)] + #[case::no_wildcard_child("/scim/v2/Users", "/scim/v2", false)] + fn wildcard(#[case] route: &str, #[case] pattern: &str, #[case] expected: bool) { + assert_eq!(matches_wildcard(route, pattern), expected); + } + + #[rstest] + #[case::group("info_routes", "/key/info", true)] + #[case::group_template("openai_routes", "/engines/gpt/chat/completions", true)] + #[case::exact_route("/v1/messages", "/v1/messages", true)] + #[case::wildcard_prefix("/scim/v2/*", "/scim/v2/Users/1", true)] + #[case::wildcard_other("/scim/v2/*", "/scim/v3", false)] + #[case::unknown_name_is_exact("not_a_group", "/not_a_group", false)] + #[case::path_swallows_a_colon_like_compile_path( + "google_routes", + "/v1beta/models/a:b:generateContent", + true + )] + #[case::path_may_be_empty_like_compile_path( + "google_routes", + "/v1beta/models/:generateContent", + true + )] + #[case::path_stops_at_a_newline_like_compile_path( + "google_routes", + "/v1beta/models/a\nb:generateContent", + false + )] + fn allowed_routes(#[case] allowed: &str, #[case] route: &str, #[case] expected: bool) { + assert_eq!(route_groups().is_route_allowed(route, &[allowed]), expected); + } + + #[test] + fn a_table_without_a_group_the_core_reads_is_refused() { + let table = route_table() + .into_iter() + .filter(|(name, _)| name != "public_routes"); + assert_eq!( + RouteGroups::new(table).unwrap_err(), + MissingRouteGroup { + group: RouteGroup::Public + } + ); + } + + #[test] + fn actual_routes_expand_groups_in_order() { + assert_eq!( + route_groups().actual_routes(&["/first", "master_key_only_routes", "/last"]), + [ + "/first", + "/global/spend/reset", + "/memory-usage-in-mem-cache", + "/memory-usage-in-mem-cache-items", + "/last", + ] + ); + } +} diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/managed.rs b/litellm/rust/crates/gateway-auth-core/src/routes/managed.rs new file mode 100644 index 00000000000..345f3e5911f --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/managed.rs @@ -0,0 +1,128 @@ +use http::Method; + +use super::{ + groups::RouteGroups, + table::{ + MANAGED_DISCOVERY_ROUTES, MANAGED_LIVE_ROUTES, MANAGED_MCP_EXCLUDED, + MANAGED_MODEL_EXTRA_ROUTES, MANAGED_MODEL_OPERATIONS, MANAGED_MODEL_PATHS, + MANAGED_MODEL_PREFIXES, MANAGED_REALTIME_ROUTES, RouteGroup, + }, +}; + +impl RouteGroups { + fn is_managed_tool_route(&self, route: &str) -> bool { + let mcp = self + .entries(RouteGroup::McpInference) + .into_iter() + .flat_map(|entries| { + entries + .routes + .iter() + .zip(&entries.templates) + .filter(|(entry, _)| !MANAGED_MCP_EXCLUDED.contains(&entry.as_str())) + }); + { mcp }.any(|(_, template)| template.matches(route)) + || self.matches(route, RouteGroup::AgentInference) + } + + pub fn managed_route(&self, route: &str) -> ManagedRoute { + if MANAGED_DISCOVERY_ROUTES.contains(&route) { + return ManagedRoute::Discovery; + } + if MANAGED_REALTIME_ROUTES.contains(&route) { + return ManagedRoute::Realtime; + } + if MANAGED_LIVE_ROUTES.contains(&route) { + return ManagedRoute::Live; + } + if is_managed_model_route(route) { + return ManagedRoute::Model; + } + if self.is_managed_tool_route(route) { + return ManagedRoute::Tool; + } + ManagedRoute::Other + } + + /// `managed_agent_route_allowed`: agent discovery is read-only, realtime and live sessions open + /// with a GET, inference is a POST, and MCP and agent-to-agent calls take any method. + pub fn managed_agent_route_allowed(&self, route: &str, method: Option<&Method>) -> bool { + match self.managed_route(route) { + ManagedRoute::Discovery => { + matches!(method, None | Some(&Method::GET | &Method::HEAD)) + } + ManagedRoute::Realtime | ManagedRoute::Live => { + matches!(method, None | Some(&Method::GET)) + } + ManagedRoute::Model => matches!(method, None | Some(&Method::POST)), + ManagedRoute::Tool => true, + ManagedRoute::Other => false, + } + } +} + +fn is_managed_model_route(route: &str) -> bool { + MANAGED_MODEL_EXTRA_ROUTES.contains(&route) + || MANAGED_MODEL_PREFIXES.iter().any(|prefix| { + route + .strip_prefix(prefix) + .and_then(|rest| rest.strip_prefix('/')) + .is_some_and(|operation| MANAGED_MODEL_OPERATIONS.contains(&operation)) + }) + || MANAGED_MODEL_PATHS.matches(route) +} + +/// The kind of route a managed agent identity calls, which decides the methods it may use and how +/// its request names a model. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ManagedRoute { + #[strum(serialize = "discovery")] + Discovery, + #[strum(serialize = "realtime")] + Realtime, + #[strum(serialize = "live")] + Live, + #[strum(serialize = "model")] + Model, + /// MCP and agent-to-agent calls. + #[strum(serialize = "tool")] + Tool, + #[strum(serialize = "other")] + Other, +} + +#[cfg(test)] +mod tests { + use http::Method; + use rstest::rstest; + + use crate::test_support::route_groups; + + #[rstest] + #[case::discovery_get("/agents", Some(Method::GET), true)] + #[case::discovery_post("/v1/agents", Some(Method::POST), false)] + #[case::realtime_get("/v1/realtime", Some(Method::GET), true)] + #[case::live_post("/live/sessions", Some(Method::POST), false)] + #[case::chat_post("/v1/chat/completions", Some(Method::POST), true)] + #[case::chat_get("/chat/completions", Some(Method::GET), false)] + #[case::deployment_path("/openai/deployments/gpt/x/chat/completions", Some(Method::POST), true)] + #[case::google_verb("/v1beta/models/gemini:generateContent", None, true)] + #[case::path_model_with_newline("/engines/a\nb/chat/completions", Some(Method::POST), true)] + #[case::empty_path_model("/engines//completions", Some(Method::POST), false)] + #[case::google_model_with_colon("/models/a:b:generateContent", Some(Method::POST), false)] + #[case::google_second_verb("/models/a:generateContent:countTokens", Some(Method::POST), false)] + #[case::mcp_any_method("/mcp/tools/call", Some(Method::DELETE), true)] + #[case::mcp_token_endpoint("/token", Some(Method::POST), false)] + #[case::a2a("/a2a/agent-1/message/send", Some(Method::POST), true)] + #[case::management("/key/generate", Some(Method::POST), false)] + fn managed_agents_reach_inference_tools_and_discovery_only( + #[case] route: &str, + #[case] method: Option, + #[case] allowed: bool, + ) { + assert_eq!( + route_groups().managed_agent_route_allowed(route, method.as_ref()), + allowed + ); + } +} diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/mod.rs b/litellm/rust/crates/gateway-auth-core/src/routes/mod.rs new file mode 100644 index 00000000000..b7409e3edc0 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/mod.rs @@ -0,0 +1,12 @@ +mod error; +mod groups; +mod managed; +pub mod passthrough; +pub(crate) mod table; +mod template; + +pub use error::MissingRouteGroup; +pub use groups::{RouteAllowlist, RouteGroups, matches_wildcard}; +pub use managed::ManagedRoute; +pub use table::RouteGroup; +pub(crate) use template::RouteSet; diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/passthrough.rs b/litellm/rust/crates/gateway-auth-core/src/routes/passthrough.rs new file mode 100644 index 00000000000..c1b07da67f6 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/passthrough.rs @@ -0,0 +1,271 @@ +//! Grants for pass-through endpoints registered with `auth: true`. `RouteChecks` calls these through +//! `_native`, so the core and the Python key path share one implementation. + +use litellm_config::RouteList; + +use super::{RouteAllowlist, matches_wildcard}; + +/// A `denied_passthrough_routes` entry that matched decides `Denied`; with none, `Denied` means +/// nothing granted the route. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PassthroughAccess { + Allowed, + Denied { denied_route: Option }, +} + +/// A team's `allowed_passthrough_routes` and `denied_passthrough_routes` metadata. `allowed` is +/// `None` when the key is missing or `None`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PassthroughGrants { + pub allowed: Option, + pub denied: Option, +} + +/// `InitPassThroughEndpointHelpers.forwarded_routes(route)`: every route the request is forwarded as. +#[derive(Clone, Copy, Debug, Default)] +pub struct ForwardedRoutes<'a>(pub &'a [String]); + +/// A key's and its team's `allowed_passthrough_routes`. +#[derive(Clone, Copy, Debug, Default)] +pub struct AllowedPassthrough<'a> { + pub key: Option<&'a RouteList>, + pub team: Option<&'a RouteList>, +} + +impl AllowedPassthrough<'_> { + /// `check_passthrough_route_access`: the key's list when it lists any, otherwise the team's. + /// A malformed key list is chosen over the team's and grants nothing. + pub fn grants(&self, route: &str) -> bool { + self.key + .filter(|allowed| !matches!(allowed, RouteList::Valid(routes) if routes.is_empty())) + .or(self.team) + .map(RouteList::items) + .unwrap_or_default() + .iter() + .any(|allowed| matches_prefix(route, allowed)) + } +} + +/// What a malformed `denied_passthrough_routes` reports as its match. It denies every route, so a +/// deny list that does not read never lets a route through. +pub const MALFORMED_DENIED_ROUTES: &str = "(not a list of routes)"; + +/// `denied_passthrough_routes` from a key's and a team's `metadata`, in that order +#[derive(Clone, Copy, Debug, Default)] +pub struct DeniedPassthrough<'a>(pub &'a [RouteList]); + +impl<'a> DeniedPassthrough<'a> { + /// `matching_denied_passthrough_route`: the first entry that matches any route the request is + /// forwarded as. + pub fn first_match(&self, forwarded: ForwardedRoutes<'_>) -> Option<&'a str> { + self.0.iter().find_map(|list| match list { + RouteList::Malformed => Some(MALFORMED_DENIED_ROUTES), + RouteList::Valid(routes) => routes.iter().map(String::as_str).find(|denied| { + forwarded + .0 + .iter() + .any(|route| matches_denied(route, denied)) + }), + }) + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct PassthroughRequest<'a> { + pub route: &'a str, + /// `RouteChecks.is_auth_enforced_pass_through_route(route, method)` + pub auth_enforced: bool, + pub forwarded_routes: ForwardedRoutes<'a>, + /// `litellm_jwtauth.team_allowed_routes` + pub jwt_team_allowed_routes: Option>, +} + +impl PassthroughRequest<'_> { + /// `team_has_passthrough_route_access`, naming the deny entry that matched when it refuses. + pub fn team_access(&self, team: &PassthroughGrants) -> PassthroughAccess { + if !self.auth_enforced { + return PassthroughAccess::Allowed; + } + let denied_route = + DeniedPassthrough(team.denied.as_slice()).first_match(self.forwarded_routes); + let team_grants = AllowedPassthrough { + key: None, + team: team.allowed.as_ref(), + }; + if denied_route.is_none() + && (self + .jwt_team_allowed_routes + .is_some_and(|routes| routes.grants_pass_through(self.route)) + || team_grants.grants(self.route)) + { + return PassthroughAccess::Allowed; + } + PassthroughAccess::Denied { + denied_route: denied_route.map(str::to_owned), + } + } +} + +/// `_route_matches_allowed_route`: the route itself or anything under it, never a sibling that +/// only shares the prefix. +pub fn matches_prefix(route: &str, allowed_route: &str) -> bool { + route + .strip_prefix(allowed_route) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('/')) +} + +/// `_route_matches_denied_route`. A `/` entry denies every route. +pub fn matches_denied(route: &str, denied_route: &str) -> bool { + let normalized = denied_route.trim_end_matches('/'); + normalized.is_empty() + || matches_prefix(route, normalized) + || matches_wildcard(route, denied_route) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::test_support::route_groups; + + fn owned(routes: &[&str]) -> Vec { + routes.iter().map(|route| (*route).to_owned()).collect() + } + + fn list(routes: &[&str]) -> RouteList { + RouteList::Valid(owned(routes).into()) + } + + #[rstest] + #[case::exact("/proxy-6", "/proxy-6", true)] + #[case::child("/proxy-6/v1/chat", "/proxy-6", true)] + #[case::sibling("/proxy-600", "/proxy-6", false)] + #[case::shorter("/proxy", "/proxy-6", false)] + fn prefix(#[case] route: &str, #[case] allowed: &str, #[case] expected: bool) { + assert_eq!(matches_prefix(route, allowed), expected); + } + + #[rstest] + #[case::root_denies_all("/anything", "/", true)] + #[case::trailing_slash("/openai/x", "/openai/", true)] + #[case::wildcard("/openai/x", "/openai/*", true)] + #[case::sibling("/openai-x", "/openai", false)] + fn denied(#[case] route: &str, #[case] denied_route: &str, #[case] expected: bool) { + assert_eq!(matches_denied(route, denied_route), expected); + } + + #[rstest] + #[case::first_entry_that_matches_any_forwarded_route(&["/a", "/v1/x"], &["/b", "/v1/*", "/a"], Some("/v1/*"))] + #[case::none(&["/a"], &["/b"], None)] + fn first_denied( + #[case] forwarded: &[&str], + #[case] denied: &[&str], + #[case] expected: Option<&str>, + ) { + let (forwarded, denied) = (owned(forwarded), [list(denied)]); + assert_eq!( + DeniedPassthrough(&denied).first_match(ForwardedRoutes(&forwarded)), + expected + ); + } + + #[rstest] + #[case::malformed_list_denies_every_route( + [list(&["/b"]), RouteList::Malformed], + Some(MALFORMED_DENIED_ROUTES) + )] + #[case::an_earlier_match_names_its_entry([list(&["/a"]), RouteList::Malformed], Some("/a"))] + fn malformed_deny_lists_deny(#[case] denied: [RouteList; 2], #[case] expected: Option<&str>) { + let forwarded = owned(&["/a"]); + assert_eq!( + DeniedPassthrough(&denied).first_match(ForwardedRoutes(&forwarded)), + expected + ); + } + + #[rstest] + #[case::explicit(&["/openai/x"], true)] + #[case::prefix_wildcard(&["/openai/*"], true)] + #[case::blanket_star(&["*", "/*"], false)] + #[case::route_group_name(&["openai_routes"], false)] + fn jwt_team_routes(#[case] allowed: &[&str], #[case] expected: bool) { + let (groups, allowed) = (route_groups(), owned(allowed)); + assert_eq!( + groups.allowlist(&allowed).grants_pass_through("/openai/x"), + expected + ); + } + + #[rstest] + #[case::key_list_wins(Some(list(&["/a"])), Some(list(&["/openai"])), false)] + #[case::empty_key_list_falls_to_team(Some(list(&[])), Some(list(&["/openai"])), true)] + #[case::team_only(None, Some(list(&["/openai"])), true)] + #[case::neither(None, None, false)] + #[case::malformed_key_list_grants_nothing(Some(RouteList::Malformed), Some(list(&["/openai"])), false)] + #[case::malformed_team_list_grants_nothing(None, Some(RouteList::Malformed), false)] + fn allowed_routes( + #[case] key: Option, + #[case] team: Option, + #[case] expected: bool, + ) { + let allowed = AllowedPassthrough { + key: key.as_ref(), + team: team.as_ref(), + }; + assert_eq!(allowed.grants("/openai/x"), expected); + } + + fn request<'a>( + forwarded: &'a [String], + jwt_team_allowed_routes: RouteAllowlist<'a>, + ) -> PassthroughRequest<'a> { + PassthroughRequest { + route: "/openai/x", + auth_enforced: true, + forwarded_routes: ForwardedRoutes(forwarded), + jwt_team_allowed_routes: Some(jwt_team_allowed_routes), + } + } + + fn grants(allowed: Option<&[&str]>, denied: &[&str]) -> PassthroughGrants { + PassthroughGrants { + allowed: allowed.map(list), + denied: Some(list(denied)), + } + } + + #[rstest] + #[case::team_grant(grants(Some(&["/openai"]), &[]), &[], PassthroughAccess::Allowed)] + #[case::jwt_team_route_grant(grants(None, &[]), &["/openai/*"], PassthroughAccess::Allowed)] + #[case::nothing_grants(grants(None, &[]), &[], PassthroughAccess::Denied { denied_route: None })] + #[case::deny_beats_grant( + grants(Some(&["/openai"]), &["/openai/*"]), + &["/openai/*"], + PassthroughAccess::Denied { denied_route: Some("/openai/*".to_owned()) } + )] + fn team_access( + #[case] team: PassthroughGrants, + #[case] jwt_team_allowed_routes: &[&str], + #[case] expected: PassthroughAccess, + ) { + let (forwarded, allowed) = (owned(&["/openai/x"]), owned(jwt_team_allowed_routes)); + let groups = route_groups(); + assert_eq!( + request(&forwarded, groups.allowlist(&allowed)).team_access(&team), + expected + ); + } + + #[test] + fn routes_that_enforce_no_auth_are_always_allowed() { + let request = PassthroughRequest { + route: "/openai/x", + ..PassthroughRequest::default() + }; + assert_eq!( + request.team_access(&grants(None, &["/"])), + PassthroughAccess::Allowed + ); + } +} diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/table.rs b/litellm/rust/crates/gateway-auth-core/src/routes/table.rs new file mode 100644 index 00000000000..65534566119 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/table.rs @@ -0,0 +1,73 @@ +use super::RouteSet; + +/// The `LiteLLMRoutes` groups the core's own rules read. The host sends every group, and the rest +/// are reached only by name from an operator's `allowed_routes`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::VariantArray)] +pub enum RouteGroup { + #[strum(serialize = "public_routes")] + Public, + #[strum(serialize = "master_key_only_routes")] + MasterKeyOnly, + #[strum(serialize = "mcp_inference_routes")] + McpInference, + #[strum(serialize = "agent_inference_routes")] + AgentInference, +} + +pub(crate) const MANAGED_DISCOVERY_ROUTES: &[&str] = &["/agents", "/v1/agents"]; +pub(crate) const MANAGED_REALTIME_ROUTES: &[&str] = &[ + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + "/realtime/translations", + "/v1/realtime/translations", +]; +pub(crate) const MANAGED_LIVE_ROUTES: &[&str] = &[ + "/live/sessions", + "/v1/live/sessions", + "/openai/v1/live/sessions", +]; +pub(crate) const MANAGED_MODEL_PREFIXES: &[&str] = &["", "/v1"]; +pub(crate) const MANAGED_MODEL_OPERATIONS: &[&str] = &[ + "chat/completions", + "completions", + "embeddings", + "responses", + "messages", + "messages/count_tokens", + "images/generations", + "images/edits", + "audio/transcriptions", + "audio/speech", + "moderations", + "rerank", + "decisions", + "systemone", + "extraction", + "ocr", +]; +pub(crate) const MANAGED_MODEL_EXTRA_ROUTES: &[&str] = &[ + "/openai/v1/responses", + "/v2/rerank", + "/claude_code_gateway/v1/messages", + "/claude_code_gateway/v1/messages/count_tokens", + "/cursor/chat/completions", +]; +pub(crate) static MANAGED_MODEL_PATHS: RouteSet = RouteSet::new(&[ + "/engines/{model:path}/chat/completions", + "/engines/{model:path}/completions", + "/engines/{model:path}/embeddings", + "/openai/deployments/{model:path}/chat/completions", + "/openai/deployments/{model:path}/completions", + "/openai/deployments/{model:path}/embeddings", + "/openai/deployments/{model:path}/images/generations", + "/openai/deployments/{model:path}/images/edits", + "/v1beta/models/{model_name:path}:countTokens", + "/v1beta/models/{model_name:path}:generateContent", + "/v1beta/models/{model_name:path}:streamGenerateContent", + "/models/{model_name:path}:countTokens", + "/models/{model_name:path}:generateContent", + "/models/{model_name:path}:streamGenerateContent", +]); +/// MCP inference routes a managed agent may call; the OAuth token endpoints are not among them. +pub(crate) const MANAGED_MCP_EXCLUDED: &[&str] = &["/token", "/introspect"]; diff --git a/litellm/rust/crates/gateway-auth-core/src/routes/template.rs b/litellm/rust/crates/gateway-auth-core/src/routes/template.rs new file mode 100644 index 00000000000..c85d461a095 --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/routes/template.rs @@ -0,0 +1,173 @@ +use std::sync::OnceLock; + +use litellm_starlette::{Convertor, Param, RouteTemplate}; + +/// A route template matched as `RouteChecks._route_matches_pattern` matches it: every parameter +/// takes one or more characters up to a `/`, whatever its convertor, and a `{name:path}` one or +/// more of any character, or of any but `:` when the template has a `:` after it. Literal text is +/// compared exactly. +#[derive(Debug)] +pub(super) struct Template(RouteTemplate); + +impl Template { + pub(super) fn compile(template: &str) -> Self { + Self(RouteTemplate::parse(template)) + } + + pub(super) fn matches(&self, route: &str) -> bool { + self.0.matches_with(route, |param, rest| { + let limit = match param { + Param { + convertor: Convertor::Path, + colon_after: true, + } => rest.find(':').unwrap_or(rest.len()), + Param { + convertor: Convertor::Path, + colon_after: false, + } => rest.len(), + Param { .. } => rest.find('/').unwrap_or(rest.len()), + }; + Box::new( + rest.char_indices() + .map(|(index, _)| index) + .chain([rest.len()]) + .filter(move |index| *index >= 1 && *index <= limit), + ) + }) + } + + /// The route matches the regex starlette's `compile_path` builds from the template, the + /// matcher `_allowed_routes_check` reads route groups with. + pub(super) fn matches_compiled(&self, route: &str) -> bool { + self.0.matches(route) + } +} + +/// Route templates named in the core, compiled on first use. +#[derive(Debug)] +pub(crate) struct RouteSet { + templates: &'static [&'static str], + compiled: OnceLock>, +} + +impl RouteSet { + pub(crate) const fn new(templates: &'static [&'static str]) -> Self { + Self { + templates, + compiled: OnceLock::new(), + } + } + + pub(crate) fn matches(&self, route: &str) -> bool { + self.compiled + .get_or_init(|| { + self.templates + .iter() + .map(|template| Template::compile(template)) + .collect() + }) + .iter() + .any(|template| template.matches(route)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::exact("/models", "/models", true)] + #[case::exact_is_whole_route("/models", "/models/x", false)] + #[case::segment("/files/{file_id}", "/files/abc", true)] + #[case::segment_needs_a_char("/files/{file_id}/content", "/files//content", false)] + #[case::segment_stops_at_slash("/files/{file_id}", "/files/a/b", false)] + #[case::path_spans_slashes( + "/v1beta/models/{model_name:path}:generateContent", + "/v1beta/models/a/b:generateContent", + true + )] + #[case::path_needs_a_char("/engines/{model:path}/completions", "/engines//completions", false)] + #[case::path_spans_newlines( + "/engines/{model:path}/chat/completions", + "/engines/a\nb/chat/completions", + true + )] + #[case::path_takes_colons_without_one_after( + "/v1/mcp/server/{path:path}", + "/v1/mcp/server/a:b", + true + )] + #[case::path_stops_before_the_templates_colon( + "/models/{model_name:path}:generateContent", + "/models/a:b:generateContent", + false + )] + #[case::path_never_swallows_another_verb( + "/models/{model_name:path}:countTokens", + "/models/a:generateContent:countTokens", + false + )] + #[case::path_before_colon_needs_a_char( + "/models/{model_name:path}:generateContent", + "/models/:generateContent", + false + )] + #[case::literal_after_param( + "/claude-code/plugins/{plugin_name}/archive/{digest}.zip", + "/claude-code/plugins/p/archive/d.zip", + true + )] + #[case::dot_is_literal( + "/claude-code/plugins/{plugin_name}/archive/{digest}.zip", + "/claude-code/plugins/p/archive/dxzip", + false + )] + #[case::question_mark_is_literal("/realtime?{model}", "/realtime?gpt", true)] + #[case::trailing_newline("/models", "/models\n", true)] + #[case::star_is_literal_in_templates("/a/*", "/a/b", false)] + #[case::host_template_drops_port("chat_completion:80", "chat_completion", true)] + #[case::int_param_is_one_segment("/custom/{item_id:int}", "/custom/5", true)] + #[case::int_param_takes_any_segment("/custom/{item_id:int}", "/custom/abc", true)] + #[case::int_param_stops_at_slash("/custom/{item_id:int}", "/custom/5/x", false)] + #[case::float_param_is_one_segment("/x/{value:float}/y", "/x/1.5/y", true)] + #[case::uuid_param_is_one_segment( + "/x/{id:uuid}", + "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca427", + true + )] + #[case::one_trailing_newline_only("/models", "/models\n\n", false)] + #[case::segment_takes_non_ascii("/files/{file_id}", "/files/\u{e9}", true)] + #[case::two_segments("/a/{x}/b/{y}", "/a/1/b/2", true)] + #[case::adjacent_params_need_a_char_each("/a/{x}{y}", "/a/b", false)] + #[case::adjacent_params_split("/a/{x}{y}", "/a/bc", true)] + #[case::int_param_backtracks_for_a_literal("/x/{id:int}5", "/x/125", true)] + #[case::two_path_params_split("/{a:path}/{b:path}/end", "/x/y/z/end", true)] + #[case::two_path_params_need_the_suffix("/{a:path}/{b:path}/end", "/x/y/z/ends", false)] + #[case::two_path_params_need_a_char_each("/{a:path}/{b:path}/end", "//end", false)] + fn templates_match_like_route_checks( + #[case] template: &str, + #[case] route: &str, + #[case] expected: bool, + ) { + assert_eq!(Template::compile(template).matches(route), expected); + } + + /// Eight `path` parameters against a route that never ends in `/end`, which trying every + /// split one at a time never finishes. + #[test] + fn many_path_params_answer_in_polynomial_time() { + let params: Vec = (0..8).map(|index| format!("{{p{index}:path}}")).collect(); + let template = Template::compile(&format!("/{}/end", params.join("/"))); + let route = format!("/{}", "x/".repeat(60)); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || sender.send(template.matches(&route))); + assert_eq!( + receiver + .recv_timeout(std::time::Duration::from_secs(10)) + .ok(), + Some(false) + ); + } +} diff --git a/litellm/rust/crates/gateway-auth-core/src/test_support.rs b/litellm/rust/crates/gateway-auth-core/src/test_support.rs new file mode 100644 index 00000000000..defbdd7e72f --- /dev/null +++ b/litellm/rust/crates/gateway-auth-core/src/test_support.rs @@ -0,0 +1,50 @@ +use crate::routes::RouteGroups; + +pub fn route_groups() -> RouteGroups { + RouteGroups::new(route_table()).expect("the stand-in table holds every group the core reads") +} + +/// A small stand-in for the `LiteLLMRoutes` table the Python host sends. +pub fn route_table() -> Vec<(String, Vec)> { + let group = |name: &str, routes: &[&str]| { + ( + name.to_owned(), + routes.iter().map(|route| (*route).to_owned()).collect(), + ) + }; + vec![ + group("public_routes", &["/", "/health/liveliness"]), + group( + "master_key_only_routes", + &[ + "/global/spend/reset", + "/memory-usage-in-mem-cache", + "/memory-usage-in-mem-cache-items", + ], + ), + group( + "openai_routes", + &[ + "/chat/completions", + "/v1/chat/completions", + "/engines/{model}/chat/completions", + "/openai/deployments/{model}/chat/completions", + ], + ), + group("info_routes", &["/key/info", "/user/info", "/team/list"]), + group( + "mcp_routes", + &["/mcp/tools/list", "/mcp/tools/call", "/token"], + ), + group( + "mcp_inference_routes", + &["/mcp/{subpath}", "/mcp/tools/call", "/token", "/introspect"], + ), + group("agent_inference_routes", &["/a2a/{agent_id}/message/send"]), + group("spend_tracking_routes", &["/spend/logs"]), + group( + "google_routes", + &["/v1beta/models/{model_name:path}:generateContent"], + ), + ] +} diff --git a/litellm/rust/crates/gateway/Cargo.toml b/litellm/rust/crates/gateway/Cargo.toml index bea1ea1a70c..331cdf8c515 100644 --- a/litellm/rust/crates/gateway/Cargo.toml +++ b/litellm/rust/crates/gateway/Cargo.toml @@ -20,6 +20,7 @@ futures-util.workspace = true litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-config.workspace = true +litellm-config-load.workspace = true litellm-core-utils.workspace = true litellm-error.workspace = true litellm-circuit-breaker.workspace = true diff --git a/litellm/rust/crates/gateway/src/error.rs b/litellm/rust/crates/gateway/src/error.rs index 779656fb76c..9683548392d 100644 --- a/litellm/rust/crates/gateway/src/error.rs +++ b/litellm/rust/crates/gateway/src/error.rs @@ -11,7 +11,7 @@ pub enum Error { #[error(transparent)] Mcp(#[from] litellm_gateway_mcp::ConnectError), #[error(transparent)] - Config(#[from] litellm_config::Error), + Config(#[from] litellm_config_load::Error), #[error(transparent)] Redis(#[from] litellm_redis_resolve::Error), #[error("Redis setting {setting} is not plain data")] diff --git a/litellm/rust/crates/gateway/src/main.rs b/litellm/rust/crates/gateway/src/main.rs index 176bc68d247..be9217d9e65 100644 --- a/litellm/rust/crates/gateway/src/main.rs +++ b/litellm/rust/crates/gateway/src/main.rs @@ -3,7 +3,6 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use litellm_config::Config; use litellm_gateway_auth::UiBackend; use litellm_tracing::{Level, Logger, Metadata, Record, Sink}; use serde_json::json; @@ -58,7 +57,7 @@ async fn main() -> Result<(), Box> { let settings: Settings = settings::from_iter(std::env::vars_os())?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; - let config = Config::load(settings.litellm_config)?; + let config = litellm_config_load::load(settings.litellm_config)?; let inference = litellm_gateway::build_inference(&config)?; let _redis = litellm_gateway::build_redis(&config, &inference.resources.pool, &inference.http).await?; diff --git a/litellm/rust/crates/python-bridge/Cargo.toml b/litellm/rust/crates/python-bridge/Cargo.toml index 8d7d714cd60..73a62d335b9 100644 --- a/litellm/rust/crates/python-bridge/Cargo.toml +++ b/litellm/rust/crates/python-bridge/Cargo.toml @@ -65,7 +65,11 @@ litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true litellm-router-core.workspace = true +litellm-config.workspace = true +litellm-gateway-auth-core.workspace = true +http.workspace = true litellm-host-python.workspace = true +parking_lot.workspace = true litellm-model-catalog.workspace = true litellm-model-catalog-refresh.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } diff --git a/litellm/rust/crates/python-bridge/src/auth/convert.rs b/litellm/rust/crates/python-bridge/src/auth/convert.rs new file mode 100644 index 00000000000..ad8c74f8e1c --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/auth/convert.rs @@ -0,0 +1,8 @@ +use http::Method; +use pyo3::{exceptions::PyValueError, prelude::*}; + +/// Upper-cased, as `RouteChecks.get_request_method` hands it over. +pub(crate) fn parse_method(text: &str) -> PyResult { + Method::from_bytes(text.to_ascii_uppercase().as_bytes()) + .map_err(|error| PyValueError::new_err(format!("invalid request method {text:?}: {error}"))) +} diff --git a/litellm/rust/crates/python-bridge/src/auth/mod.rs b/litellm/rust/crates/python-bridge/src/auth/mod.rs new file mode 100644 index 00000000000..22daf244a10 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/auth/mod.rs @@ -0,0 +1,9 @@ +mod convert; +mod passthrough; +mod route_groups; + +pub(crate) use passthrough::{ + auth_allowed_routes_grant, auth_first_denied_route, auth_jwt_team_routes_grant, + auth_managed_agent_route_allowed, auth_managed_route, auth_route_matches_denied, + auth_route_matches_prefix, auth_route_matches_wildcard, +}; diff --git a/litellm/rust/crates/python-bridge/src/auth/passthrough.rs b/litellm/rust/crates/python-bridge/src/auth/passthrough.rs new file mode 100644 index 00000000000..a79a932a157 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/auth/passthrough.rs @@ -0,0 +1,97 @@ +use litellm_config::RouteList; +use litellm_gateway_auth_core::{ + matches_wildcard, + routes::passthrough::{self, AllowedPassthrough, DeniedPassthrough, ForwardedRoutes}, +}; +use litellm_host_python::from_py; +use pyo3::prelude::*; + +use super::{convert::parse_method, route_groups::route_groups}; + +#[pyfunction] +pub(crate) fn auth_route_matches_wildcard(route: &str, pattern: &str) -> bool { + matches_wildcard(route, pattern) +} + +#[pyfunction] +pub(crate) fn auth_route_matches_prefix(route: &str, allowed_route: &str) -> bool { + passthrough::matches_prefix(route, allowed_route) +} + +#[pyfunction] +pub(crate) fn auth_route_matches_denied(route: &str, denied_route: &str) -> bool { + passthrough::matches_denied(route, denied_route) +} + +/// `denied_sources` are the raw `denied_passthrough_routes` metadata values, key first. +#[pyfunction] +pub(crate) fn auth_first_denied_route( + forwarded_routes: Vec, + denied_sources: Vec>, +) -> PyResult> { + let denied = denied_sources + .iter() + .map(route_list) + .filter_map(Result::transpose) + .collect::>>()?; + Ok(DeniedPassthrough(&denied) + .first_match(ForwardedRoutes(&forwarded_routes)) + .map(str::to_owned)) +} + +#[pyfunction] +pub(crate) fn auth_jwt_team_routes_grant( + py: Python<'_>, + route: &str, + #[pyo3(from_py_with = route_list)] team_allowed_routes: Option, +) -> PyResult { + let entries = team_allowed_routes + .as_ref() + .map(RouteList::items) + .unwrap_or_default(); + Ok(route_groups(py)? + .allowlist(entries) + .grants_pass_through(route)) +} + +/// A raw `allowed_passthrough_routes` or `denied_passthrough_routes` metadata value. A falsy one +/// is absent, as `metadata.get(...) or ...` reads it. One that does not read as a route list is +/// `Malformed`, so it denies instead of failing the request. +pub(crate) fn route_list(value: &Bound<'_, PyAny>) -> PyResult> { + Ok(value + .is_truthy()? + .then(|| from_py(value).unwrap_or(RouteList::Malformed))) +} + +/// `key` and `team` are the raw `allowed_passthrough_routes` metadata values. +#[pyfunction] +#[pyo3(signature = (route, *, key, team))] +pub(crate) fn auth_allowed_routes_grant( + route: &str, + #[pyo3(from_py_with = route_list)] key: Option, + #[pyo3(from_py_with = route_list)] team: Option, +) -> bool { + AllowedPassthrough { + key: key.as_ref(), + team: team.as_ref(), + } + .grants(route) +} + +/// The kind of route a managed agent identity calls: `discovery`, `realtime`, `live`, `model`, +/// `tool` or `other`. +#[pyfunction] +pub(crate) fn auth_managed_route(py: Python<'_>, route: &str) -> PyResult<&'static str> { + Ok(route_groups(py)?.managed_route(route).into()) +} + +#[pyfunction] +#[pyo3(signature = (route, method = None))] +pub(crate) fn auth_managed_agent_route_allowed( + py: Python<'_>, + route: &str, + method: Option<&str>, +) -> PyResult { + let method = method.map(parse_method).transpose()?; + Ok(route_groups(py)?.managed_agent_route_allowed(route, method.as_ref())) +} diff --git a/litellm/rust/crates/python-bridge/src/auth/route_groups.rs b/litellm/rust/crates/python-bridge/src/auth/route_groups.rs new file mode 100644 index 00000000000..22b6a804021 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/auth/route_groups.rs @@ -0,0 +1,34 @@ +use litellm_gateway_auth_core::RouteGroups; +use parking_lot::Mutex; +use pyo3::{exceptions::PyRuntimeError, prelude::*}; + +const MODULE: &str = "litellm.rust_bridge.host.route_groups"; + +/// The groups last built, beside the `route_groups()` value they were built from. +static BUILT: Mutex, RouteGroups)>> = Mutex::new(None); + +/// `route_groups()` as the core reads it. `LiteLLMRoutes.openai_routes` grows while the proxy runs, +/// as each auth-enforced pass-through endpoint registers, so the groups are rebuilt whenever the +/// Python value differs from the one they were built from. +pub(crate) fn route_groups(py: Python<'_>) -> PyResult { + let current = py.import(MODULE)?.call_method0("route_groups")?; + let cached = BUILT + .lock() + .as_ref() + .map(|(source, groups)| (source.clone_ref(py), groups.clone())); + if let Some((source, groups)) = cached + && source.bind(py).eq(¤t)? + { + return Ok(groups); + } + let groups = RouteGroups::new( + current + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract::<(String, Vec)>()) + .collect::>>()?, + ) + .map_err(|missing| PyRuntimeError::new_err(missing.to_string()))?; + *BUILT.lock() = Some((current.unbind(), groups.clone())); + Ok(groups) +} diff --git a/litellm/rust/crates/python-bridge/src/lib.rs b/litellm/rust/crates/python-bridge/src/lib.rs index 83b1e253c08..6b4886e8970 100644 --- a/litellm/rust/crates/python-bridge/src/lib.rs +++ b/litellm/rust/crates/python-bridge/src/lib.rs @@ -1,3 +1,4 @@ +mod auth; mod budgets; mod cache; mod callable; @@ -27,6 +28,12 @@ mod _native { use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; use pyo3::{prelude::*, types::PyModule}; + #[pymodule_export] + use crate::auth::{ + auth_allowed_routes_grant, auth_first_denied_route, auth_jwt_team_routes_grant, + auth_managed_agent_route_allowed, auth_managed_route, auth_route_matches_denied, + auth_route_matches_prefix, auth_route_matches_wildcard, + }; #[pymodule_export] use crate::budgets::NativeBudgets; #[pymodule_export] @@ -215,6 +222,14 @@ mod tests { "NativeBudgets", "budget_check_decision", "NativeS3Log", + "auth_allowed_routes_grant", + "auth_first_denied_route", + "auth_jwt_team_routes_grant", + "auth_managed_agent_route_allowed", + "auth_managed_route", + "auth_route_matches_denied", + "auth_route_matches_prefix", + "auth_route_matches_wildcard", ]; #[cfg(feature = "huggingface")] expected.push("HuggingFaceEncoding"); diff --git a/litellm/rust/crates/python-compat/AGENTS.md b/litellm/rust/crates/python-compat/AGENTS.md index 7f8185d04d0..2a1e59b1307 100644 --- a/litellm/rust/crates/python-compat/AGENTS.md +++ b/litellm/rust/crates/python-compat/AGENTS.md @@ -23,3 +23,4 @@ - Decoders recurse, so they reject nesting beyond `MAX_DEPTH` for stack safety: deliberately stricter than CPython, whose parser takes ~200 levels and whose unpickler has no limit (pinned as `nested_150`) - Formatters (`repr`, `json`) are unbounded. Values from the decoders are already capped, a hand-built `Value` is the caller's responsibility - `literal_eval` must stay linear in depth: `tests/limits.rs` times the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench +- `lenient` holds how Python LiteLLM reads a config or metadata value it never validates, one rule per Python expression (`value is True`, `bool(value)`, a value of the expected shape). A rule never fails, since the Python read never raises. Config types use them as `serde_with` adapters on plain fields, so the gateway's YAML load and python-bridge's `from_py` share one implementation. Each test case takes its expected value from that expression in CPython diff --git a/litellm/rust/crates/python-compat/Cargo.toml b/litellm/rust/crates/python-compat/Cargo.toml index 2ab6fb29843..1683bc47844 100644 --- a/litellm/rust/crates/python-compat/Cargo.toml +++ b/litellm/rust/crates/python-compat/Cargo.toml @@ -12,6 +12,7 @@ num-traits = "0.2" serde.workspace = true serde-pickle = "1.2" serde_json = { workspace = true, features = ["preserve_order"] } +serde_with.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm/rust/crates/python-compat/src/lenient.rs b/litellm/rust/crates/python-compat/src/lenient.rs new file mode 100644 index 00000000000..39c7cb02d00 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/lenient.rs @@ -0,0 +1,151 @@ +//! How Python LiteLLM reads a config or metadata value it never validates: each rule is the Python +//! expression it mirrors, and none of them fails, since the Python read never raises either. +//! +//! | Rule | Python expression | +//! |---|---| +//! | [`IsTrue`] | `value is True` | +//! | [`Truthy`] | `bool(value)` | +//! | [`Lenient`] | the value when it has the expected shape, otherwise ignored | +//! +//! The bool rules are `serde_with` adapters, used as `#[serde_as(as = "IsTrue")]` on a `bool`. + +use std::fmt; + +use serde::{ + Deserialize, Deserializer, + de::{Error, IgnoredAny, MapAccess, SeqAccess, Visitor}, +}; +use serde_with::DeserializeAs; + +/// `value is True`: only a boolean `true` is true. `"true"`, `1` and anything else are false. +pub struct IsTrue; + +impl<'de> DeserializeAs<'de, bool> for IsTrue { + fn deserialize_as>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Bool(bool), + Other(IgnoredAny), + } + + Ok(matches!(Raw::deserialize(deserializer)?, Raw::Bool(true))) + } +} + +/// `bool(value)` for plain data: `None`, `False`, zero, and empty strings and containers are false, +/// as [`crate::truthy::truthy`] reads a [`crate::Value`]. +pub struct Truthy; + +impl<'de> DeserializeAs<'de, bool> for Truthy { + fn deserialize_as>(deserializer: D) -> Result { + deserializer.deserialize_any(TruthyVisitor) + } +} + +struct TruthyVisitor; + +impl<'de> Visitor<'de> for TruthyVisitor { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("any value") + } + + fn visit_bool(self, value: bool) -> Result { + Ok(value) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(value != 0) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(value != 0) + } + + fn visit_i128(self, value: i128) -> Result { + Ok(value != 0) + } + + fn visit_u128(self, value: u128) -> Result { + Ok(value != 0) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(value != 0.0) + } + + fn visit_str(self, value: &str) -> Result { + Ok(!value.is_empty()) + } + + fn visit_bytes(self, value: &[u8]) -> Result { + Ok(!value.is_empty()) + } + + fn visit_none(self) -> Result { + Ok(false) + } + + fn visit_unit(self) -> Result { + Ok(false) + } + + fn visit_some>(self, deserializer: D) -> Result { + deserializer.deserialize_any(self) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let non_empty = seq.next_element::()?.is_some(); + while seq.next_element::()?.is_some() {} + Ok(non_empty) + } + + fn visit_map>(self, mut map: A) -> Result { + let non_empty = map.next_entry::()?.is_some(); + while map.next_entry::()?.is_some() {} + Ok(non_empty) + } +} + +/// A value Python only uses when it has the shape `T`, such as a list of route strings. Any +/// other value is `Malformed` rather than an error, and the caller decides what that grants. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(from = "RawLenient", bound(deserialize = "T: Deserialize<'de>"))] +pub enum Lenient { + Valid(T), + Malformed, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum RawLenient { + Valid(T), + Malformed(IgnoredAny), +} + +impl From> for Lenient { + fn from(raw: RawLenient) -> Self { + match raw { + RawLenient::Valid(value) => Self::Valid(value), + RawLenient::Malformed(_) => Self::Malformed, + } + } +} + +impl Lenient { + pub fn valid(&self) -> Option<&T> { + match self { + Self::Valid(value) => Some(value), + Self::Malformed => None, + } + } +} + +impl Lenient> { + /// The list's items, none when malformed. + pub fn items(&self) -> &[T] { + self.valid().map(|items| &**items).unwrap_or_default() + } +} diff --git a/litellm/rust/crates/python-compat/src/lib.rs b/litellm/rust/crates/python-compat/src/lib.rs index b079f8ed555..7ae7e4543d9 100644 --- a/litellm/rust/crates/python-compat/src/lib.rs +++ b/litellm/rust/crates/python-compat/src/lib.rs @@ -17,6 +17,7 @@ //! | [`pydantic`] | `TypeAdapter(bool)` and `TypeAdapter(int)` lax validation | //! | [`urllib`] | `urllib.parse.quote(text, safe=safe)` | //! | [`datetime`] | `datetime.fromtimestamp(seconds, timezone.utc)` fields and `datetime.fromisoformat(text)` | +//! | [`lenient`] | how Python reads an unvalidated config value: `value is True`, or a value of the expected shape | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -33,6 +34,7 @@ pub mod datetime; mod error; pub mod format; pub mod json; +pub mod lenient; pub mod literal; pub mod number; pub mod pickle; diff --git a/litellm/rust/crates/python-compat/tests/lenient.rs b/litellm/rust/crates/python-compat/tests/lenient.rs new file mode 100644 index 00000000000..2951c58df77 --- /dev/null +++ b/litellm/rust/crates/python-compat/tests/lenient.rs @@ -0,0 +1,72 @@ +use litellm_python_compat::lenient::{IsTrue, Lenient, Truthy}; +use rstest::rstest; +use serde::Deserialize; +use serde_json::{Value, json}; +use serde_with::serde_as; + +#[serde_as] +#[derive(Deserialize)] +struct Flag(#[serde_as(as = "IsTrue")] bool); + +/// Expected values from CPython: `value is True` for the JSON value loaded with `json.loads`. +#[rstest] +#[case::true_(json!(true), true)] +#[case::false_(json!(false), false)] +#[case::string_true(json!("true"), false)] +#[case::one(json!(1), false)] +#[case::null(json!(null), false)] +#[case::list_with_true(json!([true]), false)] +fn is_true_is_only_the_boolean_true(#[case] value: Value, #[case] expected: bool) { + assert_eq!(serde_json::from_value::(value).unwrap().0, expected); +} + +#[serde_as] +#[derive(Deserialize)] +struct Bool(#[serde_as(as = "Truthy")] bool); + +/// Expected values from CPython: `bool(value)` for the JSON value loaded with `json.loads`. +#[rstest] +#[case::true_(json!(true), true)] +#[case::false_(json!(false), false)] +#[case::zero(json!(0), false)] +#[case::one(json!(1), true)] +#[case::zero_float(json!(0.0), false)] +#[case::half(json!(0.5), true)] +#[case::empty_string(json!(""), false)] +#[case::string_false(json!("false"), true)] +#[case::null(json!(null), false)] +#[case::empty_list(json!([]), false)] +#[case::list_of_false(json!([false]), true)] +#[case::empty_mapping(json!({}), false)] +#[case::mapping(json!({"a": null}), true)] +fn truthy_is_python_bool(#[case] value: Value, #[case] expected: bool) { + assert_eq!(serde_json::from_value::(value).unwrap().0, expected); +} + +#[rstest] +#[case::list(json!(["/a", "/b/*"]), Lenient::Valid(["/a".to_owned(), "/b/*".to_owned()].into()))] +#[case::empty(json!([]), Lenient::Valid([].into()))] +#[case::bare_string(json!("/a/*"), Lenient::Malformed)] +#[case::number(json!(5), Lenient::Malformed)] +#[case::mapping(json!({"a": 1}), Lenient::Malformed)] +#[case::non_string_entry(json!(["/a", 5, "/b/*"]), Lenient::Malformed)] +#[case::null_entry(json!(["/a", null]), Lenient::Malformed)] +fn a_value_of_another_shape_is_malformed( + #[case] value: Value, + #[case] expected: Lenient>, +) { + assert_eq!( + serde_json::from_value::>>(value).unwrap(), + expected + ); +} + +#[rstest] +#[case::listed(Lenient::Valid(["/a".to_owned()].into()), &["/a"][..])] +#[case::malformed(Lenient::Malformed, &[][..])] +fn items_of_a_malformed_list_are_none( + #[case] list: Lenient>, + #[case] expected: &[&str], +) { + assert_eq!(list.items(), expected); +} diff --git a/litellm/rust/crates/starlette/AGENTS.md b/litellm/rust/crates/starlette/AGENTS.md new file mode 100644 index 00000000000..2c16276b79d --- /dev/null +++ b/litellm/rust/crates/starlette/AGENTS.md @@ -0,0 +1,7 @@ +# starlette + +Starlette's route template syntax, for route templates Python LiteLLM registers with FastAPI and matches with starlette, such as `LiteLLMRoutes` groups and pass-through `path`s. A crate that reads one of those templates depends on this crate and never parses `{name:convertor}` itself + +- `RouteTemplate::parse` follows `starlette.routing.compile_path`: `PARAM_REGEX` names, the five `CONVERTOR_TYPES`, and the port a host template drops. It never fails, where starlette raises on an unknown convertor, because starlette refuses such a route at registration, so no live route has one. It reads as `str` +- `RouteTemplate::matches` is the regex `compile_path` builds. Other matching rules LiteLLM writes over the same templates, such as `RouteChecks._route_matches_pattern`, belong to their callers, which pass what each parameter takes to `matches_with` rather than walking `parts()` themselves, so every matcher stays polynomial however many parameters a template has +- Every case in `tests/` takes its expected value from running `bool(compile_path(template)[0].match(route))` on starlette, never from reasoning. Run it before adding or changing a case, and update the version the tests cite when the proxy's starlette changes a convertor diff --git a/litellm/rust/crates/starlette/Cargo.toml b/litellm/rust/crates/starlette/Cargo.toml new file mode 100644 index 00000000000..8f82a513859 --- /dev/null +++ b/litellm/rust/crates/starlette/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "litellm-starlette" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +strum.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm/rust/crates/starlette/src/lib.rs b/litellm/rust/crates/starlette/src/lib.rs new file mode 100644 index 00000000000..9efb795e6ad --- /dev/null +++ b/litellm/rust/crates/starlette/src/lib.rs @@ -0,0 +1,5 @@ +#![forbid(unsafe_code)] + +mod template; + +pub use template::{Convertor, Param, RouteTemplate, TemplatePart}; diff --git a/litellm/rust/crates/starlette/src/template.rs b/litellm/rust/crates/starlette/src/template.rs new file mode 100644 index 00000000000..fb924f39f1c --- /dev/null +++ b/litellm/rust/crates/starlette/src/template.rs @@ -0,0 +1,209 @@ +use std::collections::BTreeSet; + +/// A starlette path convertor, by the name a template spells it with. `{name}` is `Str`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString)] +pub enum Convertor { + #[strum(serialize = "str")] + Str, + #[strum(serialize = "path")] + Path, + #[strum(serialize = "int")] + Int, + #[strum(serialize = "float")] + Float, + #[strum(serialize = "uuid")] + Uuid, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Param { + pub convertor: Convertor, + /// The template has a `:` anywhere after this parameter, as the Google + /// `{model:path}:generateContent` routes do. + pub colon_after: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TemplatePart { + Literal(Box), + Param(Param), +} + +/// A route template such as `/threads/{thread_id}` or a pass-through `path`. Every string parses: +/// a convertor starlette does not know reads as `Str`, and a host template such as +/// `chat_completion:80` drops its port. +#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize)] +#[serde(from = "String")] +pub struct RouteTemplate(Box<[TemplatePart]>); + +impl From for RouteTemplate { + fn from(template: String) -> Self { + Self::parse(&template) + } +} + +impl RouteTemplate { + pub fn parse(template: &str) -> Self { + let is_host = !template.starts_with('/'); + let mut parts = Vec::new(); + let mut rest = template; + while let Some((before, convertor, after)) = next_param(rest) { + parts.push(TemplatePart::Literal(before.into())); + parts.push(TemplatePart::Param(Param { + convertor: convertor + .and_then(|name| name.parse().ok()) + .unwrap_or(Convertor::Str), + colon_after: after.contains(':'), + })); + rest = after; + } + let tail = if is_host { + rest.split(':').next().unwrap_or_default() + } else { + rest + }; + parts.push(TemplatePart::Literal(tail.into())); + Self( + parts + .into_iter() + .filter(|part| !matches!(part, TemplatePart::Literal(text) if text.is_empty())) + .collect(), + ) + } + + pub fn parts(&self) -> &[TemplatePart] { + &self.0 + } + + /// The route matches the regex `compile_path` builds from the template. Each convertor takes + /// what its starlette regex takes: `str` one or more of anything but `/`, `path` zero or more + /// of anything but a newline, `int` digits, `float` digits with an optional fraction, and + /// `uuid` 32 hex digits with optional dashes between the groups. + pub fn matches(&self, route: &str) -> bool { + self.matches_with(route, |param, rest| convertor_ends(param.convertor, rest)) + } + + /// The route matches with literal text compared exactly and each parameter taking any length + /// `param_ends` yields for the rest of the route. Like `re.match` against a pattern ending in + /// `$`, a trailing newline is allowed. + pub fn matches_with( + &self, + route: &str, + param_ends: impl for<'r> Fn(Param, &'r str) -> Box + 'r>, + ) -> bool { + walk(&self.0, route, ¶m_ends) + } +} + +fn is_identifier(text: &str) -> bool { + let mut chars = text.chars(); + chars + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// The next `{name}` or `{name:convertor}`, as starlette's `PARAM_REGEX` finds it: the text before +/// it, its convertor and the text after it. +fn next_param(text: &str) -> Option<(&str, Option<&str>, &str)> { + text.match_indices('{').find_map(|(start, _)| { + let end = start + text[start..].find('}')?; + let inner = &text[start + 1..end]; + let (name, convertor) = match inner.split_once(':') { + Some((name, convertor)) => (name, Some(convertor)), + None => (inner, None), + }; + (is_identifier(name) && convertor.is_none_or(is_identifier)) + .then(|| (&text[..start], convertor, &text[end + 1..])) + }) +} + +fn is_end(rest: &str) -> bool { + rest.is_empty() || rest == "\n" +} + +/// Literals never branch, so they are stripped in place. From the first parameter on, the walk +/// keeps the set of offsets the parts so far can end at instead of trying each split in turn, +/// which keeps it at parts x route length x choices per parameter however many parameters a +/// template has. +fn walk( + parts: &[TemplatePart], + route: &str, + param_ends: &impl for<'r> Fn(Param, &'r str) -> Box + 'r>, +) -> bool { + match parts.split_first() { + None => is_end(route), + Some((TemplatePart::Literal(literal), rest)) => route + .strip_prefix(&**literal) + .is_some_and(|after| walk(rest, after, param_ends)), + Some((TemplatePart::Param(_), _)) => parts + .iter() + .try_fold(BTreeSet::from([0]), |starts, part| { + let ends: BTreeSet = match part { + TemplatePart::Literal(literal) => starts + .into_iter() + .filter(|&start| route[start..].starts_with(&**literal)) + .map(|start| start + literal.len()) + .collect(), + TemplatePart::Param(param) => starts + .into_iter() + .flat_map(|start| { + param_ends(*param, &route[start..]).map(move |len| start + len) + }) + .collect(), + }; + (!ends.is_empty()).then_some(ends) + }) + .is_some_and(|ends| ends.into_iter().any(|end| is_end(&route[end..]))), + } +} + +/// Every length of `route`'s start the convertor's regex can match. +fn convertor_ends(convertor: Convertor, route: &str) -> Box + '_> { + let boundaries = move |limit: usize, min: usize| { + route + .char_indices() + .map(|(index, _)| index) + .chain([route.len()]) + .filter(move |index| *index >= min && *index <= limit) + }; + let digits = |text: &str| text.bytes().take_while(u8::is_ascii_digit).count(); + match convertor { + Convertor::Str => Box::new(boundaries(route.find('/').unwrap_or(route.len()), 1)), + Convertor::Path => Box::new(boundaries(route.find('\n').unwrap_or(route.len()), 0)), + Convertor::Int => Box::new(1..=digits(route)), + Convertor::Float => { + let whole = digits(route); + let fraction = route[whole..] + .strip_prefix('.') + .map(digits) + .unwrap_or_default(); + let fractions = (whole > 0) + .then_some(1..=fraction) + .into_iter() + .flatten() + .map(move |digits| whole + 1 + digits); + Box::new((1..=whole).chain(fractions)) + } + Convertor::Uuid => Box::new(uuid_len(route).into_iter()), + } +} + +/// `[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}`. A dash can +/// never be a hex digit, so the match length is unique. +fn uuid_len(route: &str) -> Option { + let bytes = route.as_bytes(); + [8, 4, 4, 4, 12] + .iter() + .enumerate() + .try_fold(0, |at, (group, &width)| { + let at = match (group > 0, bytes.get(at)) { + (true, Some(b'-')) => at + 1, + _ => at, + }; + bytes + .get(at..at + width) + .filter(|hex| hex.iter().all(u8::is_ascii_hexdigit)) + .map(|_| at + width) + }) +} diff --git a/litellm/rust/crates/starlette/tests/template.rs b/litellm/rust/crates/starlette/tests/template.rs new file mode 100644 index 00000000000..5e58dc2b1ca --- /dev/null +++ b/litellm/rust/crates/starlette/tests/template.rs @@ -0,0 +1,114 @@ +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use litellm_starlette::{Convertor, Param, RouteTemplate, TemplatePart}; +use rstest::rstest; + +fn literal(text: &str) -> TemplatePart { + TemplatePart::Literal(text.into()) +} + +fn param(convertor: Convertor, colon_after: bool) -> TemplatePart { + TemplatePart::Param(Param { + convertor, + colon_after, + }) +} + +#[rstest] +#[case::plain("/models", vec![literal("/models")])] +#[case::segment("/files/{file_id}", vec![literal("/files/"), param(Convertor::Str, false)])] +#[case::int("/custom/{item_id:int}", vec![literal("/custom/"), param(Convertor::Int, false)])] +#[case::float("/x/{v:float}/y", vec![literal("/x/"), param(Convertor::Float, false), literal("/y")])] +#[case::uuid("/x/{id:uuid}", vec![literal("/x/"), param(Convertor::Uuid, false)])] +#[case::unknown_convertor_is_str("/x/{id:slug}", vec![literal("/x/"), param(Convertor::Str, false)])] +#[case::path_before_colon( + "/models/{m:path}:generateContent", + vec![literal("/models/"), param(Convertor::Path, true), literal(":generateContent")] +)] +#[case::colon_after_counts_a_later_convertor( + "/a/{x:path}{y:int}", + vec![literal("/a/"), param(Convertor::Path, true), param(Convertor::Int, false)] +)] +#[case::not_a_param("/x/{1}", vec![literal("/x/{1}")])] +#[case::host_drops_port("chat_completion:80", vec![literal("chat_completion")])] +fn templates_parse(#[case] template: &str, #[case] expected: Vec) { + assert_eq!(RouteTemplate::parse(template).parts(), expected.as_slice()); +} + +/// Expected values from starlette 1.3.1: `bool(compile_path(template)[0].match(route))`. +#[rstest] +#[case::exact("/models", "/models", true)] +#[case::trailing_newline("/models", "/models\n", true)] +#[case::one_trailing_newline_only("/models", "/models\n\n", false)] +#[case::str_needs_a_char("/files/{file_id}", "/files/", false)] +#[case::str_stops_at_slash("/files/{file_id}", "/files/a/b", false)] +#[case::str_takes_a_newline("/files/{file_id}", "/files/a\nb", true)] +#[case::str_takes_non_ascii("/files/{file_id}", "/files/\u{e9}", true)] +#[case::two_str_params("/a/{x}/b/{y}", "/a/1/b/2", true)] +#[case::adjacent_str_params_need_a_char_each("/a/{x}{y}", "/a/b", false)] +#[case::adjacent_str_params_split("/a/{x}{y}", "/a/bc", true)] +#[case::path_spans_slashes("/engines/{m:path}/completions", "/engines/a/b/completions", true)] +#[case::path_may_be_empty( + "/v1beta/models/{m:path}:generateContent", + "/v1beta/models/:generateContent", + true +)] +#[case::path_swallows_colons( + "/v1beta/models/{m:path}:generateContent", + "/v1beta/models/a:b:generateContent", + true +)] +#[case::path_stops_at_a_newline( + "/engines/{m:path}/completions", + "/engines/a\nb/completions", + false +)] +#[case::two_path_params_split("/{a:path}/{b:path}/end", "/x/y/z/end", true)] +#[case::two_path_params_need_the_suffix("/{a:path}/{b:path}/end", "/x/y/z/ends", false)] +#[case::two_path_params_need_both_slashes("/{a:path}/{b:path}/end", "//end", false)] +#[case::int_takes_digits("/x/{id:int}", "/x/42", true)] +#[case::int_needs_a_digit("/x/{id:int}", "/x/", false)] +#[case::int_refuses_letters("/x/{id:int}", "/x/4a", false)] +#[case::int_refuses_non_ascii_digits("/x/{id:int}", "/x/\u{663}", false)] +#[case::int_backtracks_for_a_literal("/x/{id:int}5", "/x/125", true)] +#[case::float_with_fraction("/x/{v:float}", "/x/1.5", true)] +#[case::float_without_fraction("/x/{v:float}", "/x/1", true)] +#[case::float_needs_whole_digits("/x/{v:float}", "/x/.5", false)] +#[case::float_needs_fraction_digits("/x/{v:float}", "/x/1.", false)] +#[case::float_takes_one_fraction("/x/{v:float}/y", "/x/1.5.5/y", false)] +#[case::float_backtracks_inside_the_fraction("/x/{v:float}0", "/x/1.50", true)] +#[case::float_leaves_the_fraction_to_a_literal("/x/{v:float}.5", "/x/1.5", true)] +#[case::uuid_with_dashes("/x/{id:uuid}", "/x/1b4e28ba-2fa1-11d2-883f-0016D3CCA427", true)] +#[case::uuid_without_dashes("/x/{id:uuid}", "/x/1b4e28ba2fa111d2883f0016d3cca427", true)] +#[case::uuid_too_short("/x/{id:uuid}", "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca42", false)] +#[case::uuid_too_long("/x/{id:uuid}", "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca427a", false)] +#[case::uuid_then_literal("/x/{id:uuid}/y", "/x/1b4e28ba-2fa1-11d2-883f-0016d3cca427/y", true)] +#[case::dot_is_literal("/a/{x}.zip", "/a/bxzip", false)] +fn templates_match_like_compile_path( + #[case] template: &str, + #[case] route: &str, + #[case] expected: bool, +) { + assert_eq!(RouteTemplate::parse(template).matches(route), expected); +} + +fn answer_within(budget: Duration, check: impl FnOnce() -> bool + Send + 'static) -> Option { + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || sender.send(check())); + receiver.recv_timeout(budget).ok() +} + +/// A pass-through `path` with eight `path` parameters against a route that never ends in `/end`. +/// Trying every split one at a time never finishes. Expected value from starlette 1.3.1. +#[test] +fn many_path_params_answer_in_polynomial_time() { + let params: Vec = (0..8).map(|index| format!("{{p{index}:path}}")).collect(); + let template = RouteTemplate::parse(&format!("/{}/end", params.join("/"))); + let route = format!("/{}", "x/".repeat(60)); + assert_eq!( + answer_within(Duration::from_secs(10), move || template.matches(&route)), + Some(false) + ); +} diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index a4067705c7e..06d2d77e41e 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -523,6 +523,14 @@ class LocalSizes: @property def key_objects(self) -> int: ... +def auth_route_matches_wildcard(route: str, pattern: str) -> bool: ... +def auth_managed_route(route: str) -> Literal["discovery", "realtime", "live", "model", "tool", "other"]: ... +def auth_managed_agent_route_allowed(route: str, method: str | None = None) -> bool: ... +def auth_route_matches_prefix(route: str, allowed_route: str) -> bool: ... +def auth_route_matches_denied(route: str, denied_route: str) -> bool: ... +def auth_first_denied_route(forwarded_routes: Sequence[str], denied_sources: Sequence[object]) -> str | None: ... +def auth_jwt_team_routes_grant(route: str, team_allowed_routes: object) -> bool: ... +def auth_allowed_routes_grant(route: str, *, key: object, team: object) -> bool: ... @final class NativeAuthCache: def __new__( @@ -872,6 +880,14 @@ __all__ = [ "aocr", "aresponses", "atranscription", + "auth_allowed_routes_grant", + "auth_first_denied_route", + "auth_jwt_team_routes_grant", + "auth_managed_agent_route_allowed", + "auth_managed_route", + "auth_route_matches_denied", + "auth_route_matches_prefix", + "auth_route_matches_wildcard", "budget_check_decision", "bundled_model_cost_map", "completion", diff --git a/litellm/rust_bridge/host/AGENTS.md b/litellm/rust_bridge/host/AGENTS.md index da933f3b467..699cb1d85bb 100644 --- a/litellm/rust_bridge/host/AGENTS.md +++ b/litellm/rust_bridge/host/AGENTS.md @@ -32,6 +32,7 @@ Python behavior Rust needs but does not own, behind one module per question. Rus - `redis_events`: reporting native Redis calls and breaker transitions to `ServiceLogging` - `redis_credentials`: what a caller's redis-py `CredentialProvider` authenticates with - `model_cost`: where a `register_model` entry lands in the published table and the row it writes + - `route_groups`: the proxy's `LiteLLMRoutes` groups by name, which the auth core reads at settings build - `routes/.py` holds the questions one route asks, under the same function names on every route (`response`, `map_failure`, `connection_defaults`), so Rust names only the module. A question two routes ask goes in a top-level module here instead - Each module reuses the Python implementation the Python route runs (`AnthropicModelInfo`, `Logging`). Do not re-derive those rules here or in Rust; one resolver, two callers diff --git a/litellm/rust_bridge/host/route_groups.py b/litellm/rust_bridge/host/route_groups.py new file mode 100644 index 00000000000..a10bce92489 --- /dev/null +++ b/litellm/rust_bridge/host/route_groups.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +from litellm.proxy._types import LiteLLMRoutes + + +def route_groups() -> Mapping[str, tuple[str, ...]]: + return MappingProxyType({member.name: tuple(member.value) for member in LiteLLMRoutes}) diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 3883514e34a..56b7501dd1c 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -1884,6 +1884,49 @@ def test_check_passthrough_route_access_empty_list(): assert result is False +@pytest.mark.parametrize( + "key_routes", + [ + pytest.param("/custom-endpoint", id="bare_string"), + pytest.param(["/custom-endpoint", None], id="null_entry_after_a_match"), + pytest.param(5, id="number"), + pytest.param(object(), id="unconvertible_object"), + ], +) +def test_a_malformed_key_list_grants_nothing_and_shadows_the_team_list(key_routes: object) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="test_user", + metadata={"allowed_passthrough_routes": key_routes}, + team_metadata={"allowed_passthrough_routes": ["/custom-endpoint"]}, + ) + + assert ( + RouteChecks.check_passthrough_route_access(route="/custom-endpoint/x", user_api_key_dict=valid_token) is False + ) + + +@pytest.mark.parametrize( + "key_routes", + [ + pytest.param([], id="empty_list"), + pytest.param("", id="empty_string"), + pytest.param(0, id="zero"), + pytest.param(False, id="false"), + pytest.param({}, id="empty_dict"), + ], +) +def test_an_empty_key_list_falls_back_to_the_team_list(key_routes: object) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="test_user", + metadata={"allowed_passthrough_routes": key_routes}, + team_metadata={"allowed_passthrough_routes": ["/custom-endpoint"]}, + ) + + assert ( + RouteChecks.check_passthrough_route_access(route="/custom-endpoint/x", user_api_key_dict=valid_token) is True + ) + + @pytest.mark.parametrize( "route", [ @@ -4708,6 +4751,71 @@ def test_denied_passthrough_routes_win_over_allow( assert f"Matched `{denied_route}` in `denied_passthrough_routes`" in exc_info.value.detail +@pytest.mark.parametrize( + "denied", + [ + pytest.param(["/other", None], id="null_entry"), + pytest.param("/other", id="bare_string"), + pytest.param(5, id="number"), + ], +) +def test_a_malformed_deny_list_denies_every_route_instead_of_failing(denied: object) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + metadata={"allowed_passthrough_routes": ["/svc"]}, + team_metadata={"denied_passthrough_routes": denied}, + ) + + with pytest.raises(HTTPException) as exc_info: + _check_route_with_registered_routes(route="/svc/public", valid_token=valid_token) + + assert exc_info.value.status_code == 403 + assert "Matched `(not a list of routes)` in `denied_passthrough_routes`" in exc_info.value.detail + + +@pytest.mark.parametrize( + "denied", + [ + pytest.param([], id="empty_list"), + pytest.param("", id="empty_string"), + pytest.param(0, id="zero"), + pytest.param(False, id="false"), + pytest.param({}, id="empty_dict"), + ], +) +def test_an_empty_deny_list_denies_nothing(denied: object) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + metadata={"allowed_passthrough_routes": ["/svc"]}, + team_metadata={"denied_passthrough_routes": denied}, + ) + + assert ( + RouteChecks.matching_denied_passthrough_route( + route="/svc/public", metadata_sources=(valid_token.metadata, valid_token.team_metadata) + ) + is None + ) + _check_route_with_registered_routes(route="/svc/public", valid_token=valid_token) + + +@pytest.mark.parametrize( + ("team_allowed_routes", "expected"), + [(["/openai/*"], True), (["/openai/*", None], False), ("/openai/*", False)], + ids=["list", "null_entry", "bare_string"], +) +def test_jwt_team_routes_grant_only_from_a_list_of_routes(team_allowed_routes: object, expected: bool) -> None: + assert ( + RouteChecks.jwt_team_routes_grant_pass_through( + route="/openai/x", + team_allowed_routes=team_allowed_routes, # pyright: ignore[reportArgumentType] # malformed on purpose + ) + is expected + ) + + @pytest.mark.parametrize( "route", ["/svc/public", "/svc/administrator", "/anthropic/v1/messages", "/chat/completions"], diff --git a/tests/unit/rust_bridge/host/test_route_groups.py b/tests/unit/rust_bridge/host/test_route_groups.py new file mode 100644 index 00000000000..5ebd6cc803f --- /dev/null +++ b/tests/unit/rust_bridge/host/test_route_groups.py @@ -0,0 +1,10 @@ +from litellm.proxy._types import LiteLLMRoutes +from litellm.rust_bridge.host.route_groups import route_groups + + +def test_route_groups_hand_rust_every_litellm_route_group_by_name(): + groups = route_groups() + + assert {name: set(routes) for name, routes in groups.items()} == { + member.name: set(member.value) for member in LiteLLMRoutes + }