From 334e7119c3842281fa60ac162923f3f9aa6bb41b Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Tue, 20 Oct 2026 17:00:42 -0700 Subject: [PATCH] feat(gateway): serve /v1/models (#47640) List the model groups a caller may use at GET /v1/models and /models, in the proxy's OpenAI list shape, and answer GET /v1/models/{model_id} and /models/{model_id} with one model or the proxy's 404 A group is listed when the caller is authorized for one of its deployments the same way an inference request naming it would be: key and team model lists, all-proxy-models, team-owned deployments. Groups whose deployments all set model_info.discoverable: false are listed for the master key only, and wildcard groups are left out. openai_routes, llm_api_routes and info_routes now allow the listing routes --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/gateway-auth/src/keys.rs | 18 ++- .../crates/gateway-inference/Cargo.toml | 1 + .../crates/gateway-inference/src/error.rs | 8 + .../crates/gateway-inference/src/lib.rs | 10 +- .../crates/gateway-inference/src/models.rs | 153 ++++++++++++++++++ .../crates/gateway/tests/virtual_keys.rs | 114 +++++++++++++ litellm-rust/crates/router/src/deployment.rs | 4 + litellm-rust/crates/router/src/lib.rs | 22 +++ litellm-rust/crates/router/tests/router.rs | 26 +++ 10 files changed, 353 insertions(+), 4 deletions(-) create mode 100644 litellm-rust/crates/gateway-inference/src/models.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 9500334d979..b9b0c4c51e9 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3847,6 +3847,7 @@ dependencies = [ "litellm-auth", "litellm-cache-memory", "litellm-cache-response", + "litellm-core-utils", "litellm-gateway-auth", "litellm-gateway-hooks", "litellm-host", diff --git a/litellm-rust/crates/gateway-auth/src/keys.rs b/litellm-rust/crates/gateway-auth/src/keys.rs index 654b8503d3e..9e3c5927072 100644 --- a/litellm-rust/crates/gateway-auth/src/keys.rs +++ b/litellm-rust/crates/gateway-auth/src/keys.rs @@ -108,7 +108,7 @@ impl From for KeyModels { } /// The gateway's routes in the proxy's `openai_routes` preset. -const OPENAI_ROUTES: [&str; 18] = [ +const OPENAI_ROUTES: [&str; 20] = [ "/chat/completions", "/v1/chat/completions", "/engines/{model}/chat/completions", @@ -127,11 +127,16 @@ const OPENAI_ROUTES: [&str; 18] = [ "/v1/responses", "/ocr", "/v1/ocr", + "/models", + "/v1/models", ]; /// The gateway's routes in the proxy's `anthropic_routes` preset. const ANTHROPIC_ROUTES: [&str; 2] = ["/v1/messages", "/v1/messages/count_tokens"]; +/// The gateway's routes in the proxy's `info_routes` preset. +const INFO_ROUTES: [&str; 2] = ["/models", "/v1/models"]; + /// A key's `allowed_routes`. Empty allows every route. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct AllowedRoutes(Arc<[String]>); @@ -142,8 +147,8 @@ impl AllowedRoutes { } /// The proxy's `RouteChecks.is_virtual_key_allowed_to_call_route` for the routes this gateway - /// serves. Presets other than `llm_api_routes`, `openai_routes` and `anthropic_routes` grant - /// nothing here. + /// serves. Presets other than `llm_api_routes`, `openai_routes`, `anthropic_routes` and + /// `info_routes` grant nothing here. pub fn allows(&self, route: &str) -> bool { self.0.is_empty() || self.0.iter().any(|allowed| { @@ -170,6 +175,7 @@ fn preset(name: &str) -> impl Iterator { "llm_api_routes" => &[&OPENAI_ROUTES, &ANTHROPIC_ROUTES], "openai_routes" => &[&OPENAI_ROUTES], "anthropic_routes" => &[&ANTHROPIC_ROUTES], + "info_routes" => &[&INFO_ROUTES], _ => &[], }; routes.iter().flat_map(|group| group.iter().copied()) @@ -570,6 +576,12 @@ mod tests { #[case::llm_api_routes_not_mcp(&["llm_api_routes"], "/mcp/sse", false)] #[case::openai_routes_chat(&["openai_routes"], "/v1/chat/completions", true)] #[case::openai_routes_responses(&["openai_routes"], "/v1/responses", true)] + #[case::openai_routes_models(&["openai_routes"], "/v1/models", true)] + #[case::openai_routes_not_model_retrieval(&["openai_routes"], "/v1/models/{model_id}", false)] + #[case::llm_api_routes_models(&["llm_api_routes"], "/models", true)] + #[case::info_routes_models(&["info_routes"], "/v1/models", true)] + #[case::info_routes_not_chat(&["info_routes"], "/v1/chat/completions", false)] + #[case::exact_models_allows_retrieval(&["/v1/models"], "/v1/models/{model_id}", true)] #[case::openai_routes_not_messages(&["openai_routes"], "/v1/messages", false)] #[case::anthropic_routes_messages(&["anthropic_routes"], "/v1/messages", true)] #[case::anthropic_routes_not_chat(&["anthropic_routes"], "/chat/completions", false)] diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 5b112f791ec..abe75f0fbea 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -11,6 +11,7 @@ axum = { workspace = true, features = ["json", "original-uri"] } base64.workspace = true bytes.workspace = true litellm-auth.workspace = true +litellm-core-utils.workspace = true litellm-gateway-auth.workspace = true litellm-gateway-hooks.workspace = true litellm-inference.workspace = true diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index 230663e2dd9..a39c632bb54 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -29,6 +29,9 @@ pub enum Error { "/v1/messages: Invalid model name passed in model={0}. Call `/v1/models` to view available models for your key." )] UnknownModel(String), + /// `/v1/models/{model_id}` for a model that does not exist or that the caller may not use. + #[error("The model `{0}` does not exist or is not accessible")] + ModelNotFound(String), #[error(transparent)] Route(#[from] RouteError), #[error(transparent)] @@ -107,6 +110,7 @@ impl Error { | Self::UnknownModel(_) | Self::MissingParam { .. } | Self::InvalidMessages(_) => StatusCode::BAD_REQUEST, + Self::ModelNotFound(_) => StatusCode::NOT_FOUND, Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => { StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY) } @@ -122,6 +126,10 @@ impl Error { match self { Self::Auth(error) => error.into_response(), Self::Refused(refusal) => refused(refusal), + // The proxy raises a bare `HTTPException`, which FastAPI answers as `{"detail"}`. + Self::ModelNotFound(_) => { + (self.status(), Json(json!({ "detail": self.to_string() }))).into_response() + } other => (other.status(), Json(other.openai_body())).into_response(), } } diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index e0209446c60..0bf26ad7101 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -9,12 +9,16 @@ mod client_side; mod error; mod hooks; pub mod messages; +mod models; pub mod observe; mod request; use std::sync::{Arc, OnceLock}; -use axum::{Router, routing::post}; +use axum::{ + Router, + routing::{get, post}, +}; use litellm_http::{ ClientVariant, HttpClientConfig, media::{MediaFetcher, UrlPolicy}, @@ -144,6 +148,10 @@ pub fn router(gateway: Arc) -> Router { .route("/v1/messages", post(messages::create)) .route("/chat/completions", post(chat_completions::create)) .route("/v1/chat/completions", post(chat_completions::create)) + .route("/models", get(models::list)) + .route("/v1/models", get(models::list)) + .route("/models/{model_id}", get(models::retrieve)) + .route("/v1/models/{model_id}", get(models::retrieve)) .nest("/engines/{model}", model_routes()) .nest("/openai/deployments/{model}", model_routes()) // Not served: these routes skip the hooks (budgets, rate limits, spend), so a key could diff --git a/litellm-rust/crates/gateway-inference/src/models.rs b/litellm-rust/crates/gateway-inference/src/models.rs new file mode 100644 index 00000000000..d51eef6ea4f --- /dev/null +++ b/litellm-rust/crates/gateway-inference/src/models.rs @@ -0,0 +1,153 @@ +//! `GET /v1/models` and `GET /v1/models/{model_id}`: the model groups a caller may use, in the +//! OpenAI shape the proxy's `model_list` and `model_info` answer with. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, State}, +}; +use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; +use litellm_gateway_auth::{AuthenticatedRequest, AuthenticationMethod}; +use litellm_router::{Deployment, ModelGroup}; +use serde::Serialize; + +use crate::{Error, Gateway, request}; + +/// The proxy's `DEFAULT_MODEL_CREATED_AT_TIME`. +const CREATED: u64 = 1_677_610_602; + +#[derive(Debug, PartialEq, Serialize)] +pub(crate) struct Model { + id: String, + object: &'static str, + created: u64, + owned_by: String, +} + +impl Model { + fn new(id: &str, owned_by: &str) -> Self { + Self { + id: id.to_owned(), + object: "model", + created: CREATED, + owned_by: owned_by.to_owned(), + } + } +} + +#[derive(Debug, Serialize)] +pub(crate) struct ModelList { + data: Vec, + object: &'static str, +} + +/// One entry per model group the caller may call, in config order. The listing names every +/// model's owner `openai`, as the proxy's does. +pub(crate) async fn list( + State(gateway): State>, + identity: AuthenticatedRequest, +) -> Json { + let mut data = Vec::new(); + for group in gateway.models.groups() { + if visible(&identity, group, true).await.is_some() { + data.push(Model::new(group.name(), "openai")); + } + } + Json(ModelList { + data, + object: "list", + }) +} + +/// The model `model_id` names when the caller may call it, owned by its deployment's provider. +pub(crate) async fn retrieve( + State(gateway): State>, + identity: AuthenticatedRequest, + Path(model_id): Path, +) -> Result, Error> { + let Some(group) = gateway.models.group(&model_id) else { + return Err(Error::ModelNotFound(model_id)); + }; + match visible(&identity, group, false).await { + Some(deployment) => Ok(Json(Model::new(&model_id, provider(deployment)))), + None => Err(Error::ModelNotFound(model_id)), + } +} + +/// The first deployment of `group` the caller may be routed to and is authorized for, as a +/// request naming the group would be. When `listing`, only the master key sees a group every +/// such deployment marks `discoverable: false`; any caller allowed it still retrieves it by +/// name, as with the proxy. A wildcard group is never shown: the gateway routes model names exactly, so no +/// request reaches it under another name. +async fn visible<'a>( + identity: &AuthenticatedRequest, + group: &'a ModelGroup, + listing: bool, +) -> Option<&'a Deployment> { + if group.name().contains('*') { + return None; + } + let hide_undiscoverable = + listing && identity.caller().authentication().method != AuthenticationMethod::MasterKey; + for deployment in group.deployments() { + if !identity.may_use_deployment(deployment.team_id.as_deref()) + || (deployment.undiscoverable && hide_undiscoverable) + { + continue; + } + if request::authorize(identity, group.name(), deployment) + .await + .is_ok() + { + return Some(deployment); + } + } + None +} + +/// The deployment's provider: its `custom_llm_provider`, else its model's prefix, else +/// `openai`, which the proxy infers for a bare model name. +fn provider(deployment: &Deployment) -> &str { + get_custom_llm_provider(&deployment.model, deployment.custom_llm_provider.as_deref()) + .map_or("openai", |resolved| resolved.custom_llm_provider) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::prefix("anthropic/claude-x", None, "anthropic")] + #[case::custom_provider("claude-x", Some("bedrock"), "bedrock")] + #[case::custom_provider_wins("openai/gpt-x", Some("azure"), "azure")] + #[case::bare_name("gpt-x", None, "openai")] + fn names_the_deployments_provider( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] expected: &str, + ) { + let deployment = Deployment { + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..Deployment::default() + }; + + assert_eq!(provider(&deployment), expected); + } + + #[test] + fn serializes_the_openai_model_shape() { + assert_eq!( + serde_json::to_value(Model::new("claude-a", "openai")).unwrap(), + serde_json::json!({ + "id": "claude-a", + "object": "model", + "created": 1_677_610_602, + "owned_by": "openai", + }) + ); + } +} diff --git a/litellm-rust/crates/gateway/tests/virtual_keys.rs b/litellm-rust/crates/gateway/tests/virtual_keys.rs index 4c6d0c41957..84ff1809a83 100644 --- a/litellm-rust/crates/gateway/tests/virtual_keys.rs +++ b/litellm-rust/crates/gateway/tests/virtual_keys.rs @@ -187,6 +187,17 @@ async fn app() -> Router { api_key: sk-upstream model_info: team_id: team-2 + - model_name: claude-hidden + litellm_params: + model: anthropic/claude-x + api_base: {upstream} + api_key: sk-upstream + model_info: + discoverable: false + - model_name: openai/* + litellm_params: + model: openai/* + api_key: sk-upstream router_settings: num_retries: 0 general_settings: @@ -284,6 +295,109 @@ async fn messages_accept_virtual_keys_from_every_key_header( } } +async fn get(app: Router, path: &str, key: Option<&str>) -> (u16, Value) { + let request = Request::get(path); + let request = match key { + Some(key) => request.header("authorization", format!("Bearer {key}")), + None => request, + }; + let response = app + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1 << 20).await.unwrap(); + (status, serde_json::from_slice(&body).unwrap()) +} + +/// The OpenAI model list of `ids`, as the proxy's `/v1/models` answers. +fn model_list(ids: &[&str]) -> Value { + json!({ + "data": ids + .iter() + .map(|id| json!({"id": id, "object": "model", "created": 1_677_610_602, "owned_by": "openai"})) + .collect::>(), + "object": "list", + }) +} + +/// One entry per model group the key may call, in config order: team-owned groups only for +/// their team, `discoverable: false` groups only for the master key, and no wildcard group. +#[rstest] +#[case::master_key("sk-master", &["claude-a", "claude-owned", "claude-mixed", "claude-hidden"])] +#[case::all_models("sk-all", &["claude-a", "claude-mixed"])] +#[case::key_models("sk-scoped", &["claude-a"])] +#[case::key_model_not_deployed("sk-other", &[])] +#[case::own_team("sk-team-2", &["claude-a", "claude-owned", "claude-mixed"])] +#[case::other_team("sk-team-1", &["claude-a", "claude-mixed"])] +#[case::team_models("sk-team-narrowed", &[])] +#[case::openai_routes("sk-chat-only", &["claude-a", "claude-mixed"])] +#[case::llm_api_routes("sk-messages-only", &["claude-a", "claude-mixed"])] +#[tokio::test] +async fn models_lists_the_groups_a_key_may_call( + #[future] app: Router, + #[case] key: &str, + #[case] expected: &[&str], + #[values("/v1/models", "/models")] path: &str, +) { + let (status, body) = get(app.await, path, Some(key)).await; + + assert_eq!(status, 200, "{body}"); + assert_eq!(body, model_list(expected)); +} + +#[rstest] +#[case::missing(None, 401, "auth_error")] +#[case::unknown(Some("sk-unknown"), 401, "token_not_found_in_db")] +#[case::expired(Some("sk-expired"), 401, "expired_key")] +#[tokio::test] +async fn models_requires_a_valid_key( + #[future] app: Router, + #[case] key: Option<&str>, + #[case] status: u16, + #[case] error_type: &str, +) { + let (actual, body) = get(app.await, "/v1/models", key).await; + + assert_eq!(actual, status, "{body}"); + assert_eq!(body["error"]["type"], error_type); +} + +#[rstest] +#[case::master_key("sk-master", "claude-owned", 200)] +#[case::shared("sk-all", "claude-a", 200)] +#[case::own_team("sk-team-2", "claude-owned", 200)] +#[case::other_team("sk-team-1", "claude-owned", 404)] +#[case::key_models_exclude("sk-scoped", "claude-mixed", 404)] +#[case::team_models_exclude("sk-team-narrowed", "claude-a", 404)] +#[case::undiscoverable_retrievable_by_name("sk-all", "claude-hidden", 200)] +#[case::undiscoverable_still_needs_access("sk-scoped", "claude-hidden", 404)] +#[case::unknown_model("sk-master", "gpt-unknown", 404)] +#[case::route_preset_lists_only("sk-chat-only", "claude-a", 403)] +#[tokio::test] +async fn model_retrieval_answers_only_models_the_key_may_call( + #[future] app: Router, + #[case] key: &str, + #[case] model: &str, + #[case] status: u16, + #[values("/v1/models", "/models")] prefix: &str, +) { + let (actual, body) = get(app.await, &format!("{prefix}/{model}"), Some(key)).await; + + assert_eq!(actual, status, "{body}"); + match status { + 200 => assert_eq!( + body, + json!({"id": model, "object": "model", "created": 1_677_610_602, "owned_by": "anthropic"}) + ), + 404 => assert_eq!( + body, + json!({"detail": format!("The model `{model}` does not exist or is not accessible")}) + ), + _ => assert_eq!(body["error"]["type"], "auth_error"), + } +} + /// A Bedrock-shaped upstream counting the requests it receives: Converse calls get a Converse /// reply and invoke calls an Anthropic message. async fn bedrock_upstream() -> (String, Arc) { diff --git a/litellm-rust/crates/router/src/deployment.rs b/litellm-rust/crates/router/src/deployment.rs index d2f55061f1c..72aa15abdd5 100644 --- a/litellm-rust/crates/router/src/deployment.rs +++ b/litellm-rust/crates/router/src/deployment.rs @@ -24,6 +24,9 @@ pub struct Deployment { /// The model a deployment serves under another name, from `litellm_params.base_model` or /// `model_info.base_model`, which Azure's param rules and prices follow. pub base_model: Option, + /// `model_info.discoverable: false`: the deployment still serves requests, but only the + /// master key sees it in model listings. + pub undiscoverable: bool, } impl Deployment { @@ -49,6 +52,7 @@ impl Deployment { shaping: MessagesShaping::default(), team_id: None, id: None, + undiscoverable: false, base_model: params .extra .get("base_model") diff --git a/litellm-rust/crates/router/src/lib.rs b/litellm-rust/crates/router/src/lib.rs index 3e491c51b68..6ca6fd3feec 100644 --- a/litellm-rust/crates/router/src/lib.rs +++ b/litellm-rust/crates/router/src/lib.rs @@ -27,6 +27,8 @@ use health::Health; /// The model groups of a config and the health of each deployment in them. pub struct Router { groups: HashMap, + /// The groups' names in the order the config first names them. + names: Box<[Box]>, /// Every deployment's shared-state id and health, in a fixed order. deployments: Box<[(Box, Arc)]>, shared: OnceLock>, @@ -91,6 +93,7 @@ impl Router { .base_model .clone() .or_else(|| info_base_model(model)), + undiscoverable: undiscoverable(model), ..deployment }, )) @@ -121,6 +124,11 @@ impl Router { (name, group) }) .collect(); + let names = order + .iter() + .filter(|(_, index)| *index == 0) + .map(|(name, _)| name.as_str().into()) + .collect(); let deployments = order .into_iter() .map(|(name, index)| { @@ -130,6 +138,7 @@ impl Router { .collect(); Self { groups, + names, deployments, shared: OnceLock::new(), policy, @@ -162,6 +171,11 @@ impl Router { self.groups.get(model_name) } + /// Every model group, in the order the config first names each. + pub fn groups(&self) -> impl ExactSizeIterator { + self.names.iter().map(|name| &self.groups[&**name]) + } + pub fn policy(&self) -> &RoutingPolicy { &self.policy } @@ -367,6 +381,14 @@ fn owner_team(model: &Model) -> Option { } } +/// `model_info.discoverable: false`, which keeps the deployment out of model listings. +fn undiscoverable(model: &Model) -> bool { + matches!( + model.model_info.get("discoverable"), + Some(litellm_config::Value::Bool(false)) + ) +} + fn info_base_model(model: &Model) -> Option { match model.model_info.get("base_model")? { litellm_config::Value::String(name) if !name.is_empty() => Some(name.clone()), diff --git a/litellm-rust/crates/router/tests/router.rs b/litellm-rust/crates/router/tests/router.rs index d92438bf296..14b5e7c7194 100644 --- a/litellm-rust/crates/router/tests/router.rs +++ b/litellm-rust/crates/router/tests/router.rs @@ -118,6 +118,7 @@ fn programmatic_deployments_preserve_overrides_and_share_their_group() { team_id: None, id: None, base_model: None, + undiscoverable: false, }; let router = Router::from_iter([ ("public-model".into(), Deployment::default()), @@ -161,6 +162,31 @@ fn configuration_keeps_every_deployment_of_a_model_name_in_order() { .collect::>(), ["provider/a", "provider/b", "provider/c"] ); + assert_eq!( + router + .groups() + .map(|group| group.name()) + .collect::>(), + ["shared", "other"] + ); +} + +#[rstest] +#[case::unset("{}", false)] +#[case::discoverable("{discoverable: true}", false)] +#[case::undiscoverable("{discoverable: false}", true)] +#[case::not_a_bool("{discoverable: \"false\"}", false)] +fn configuration_reads_discoverable(#[case] model_info: &str, #[case] undiscoverable: bool) { + let config = Config::from_yaml(&format!( + "model_list: + - model_name: m + litellm_params: {{model: provider/a}} + model_info: {model_info}" + )) + .unwrap(); + let router = Router::from_config(&config).unwrap(); + + assert_eq!(only(&router, "m").undiscoverable, undiscoverable); } #[rstest]