feat(gateway): serve /v1/models (#47640)

List the model groups a caller may use at GET /v1/models and /models, in the
proxy's OpenAI list shape, and answer GET /v1/models/{model_id} and
/models/{model_id} with one model or the proxy's 404

A group is listed when the caller is authorized for one of its deployments the
same way an inference request naming it would be: key and team model lists,
all-proxy-models, team-owned deployments. Groups whose deployments all set
model_info.discoverable: false are listed for the master key only, and wildcard
groups are left out. openai_routes, llm_api_routes and info_routes now allow
the listing routes

litellm-rust/Cargo.lock

@@ -3847,6 +3847,7 @@ dependencies = [
  "litellm-auth",
  "litellm-cache-memory",
  "litellm-cache-response",
+ "litellm-core-utils",
  "litellm-gateway-auth",
  "litellm-gateway-hooks",
  "litellm-host",

litellm-rust/crates/gateway-auth/src/keys.rs

@@ -108,7 +108,7 @@ impl From<ModelAccess> for KeyModels {
 }
 
 /// The gateway's routes in the proxy's `openai_routes` preset.
-const OPENAI_ROUTES: [&str; 18] = [
+const OPENAI_ROUTES: [&str; 20] = [
     "/chat/completions",
     "/v1/chat/completions",
     "/engines/{model}/chat/completions",
@@ -127,11 +127,16 @@ const OPENAI_ROUTES: [&str; 18] = [
     "/v1/responses",
     "/ocr",
     "/v1/ocr",
+    "/models",
+    "/v1/models",
 ];
 
 /// The gateway's routes in the proxy's `anthropic_routes` preset.
 const ANTHROPIC_ROUTES: [&str; 2] = ["/v1/messages", "/v1/messages/count_tokens"];
 
+/// The gateway's routes in the proxy's `info_routes` preset.
+const INFO_ROUTES: [&str; 2] = ["/models", "/v1/models"];
+
 /// A key's `allowed_routes`. Empty allows every route.
 #[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct AllowedRoutes(Arc<[String]>);
@@ -142,8 +147,8 @@ impl AllowedRoutes {
     }
 
     /// The proxy's `RouteChecks.is_virtual_key_allowed_to_call_route` for the routes this gateway
-    /// serves. Presets other than `llm_api_routes`, `openai_routes` and `anthropic_routes` grant
-    /// nothing here.
+    /// serves. Presets other than `llm_api_routes`, `openai_routes`, `anthropic_routes` and
+    /// `info_routes` grant nothing here.
     pub fn allows(&self, route: &str) -> bool {
         self.0.is_empty()
             || self.0.iter().any(|allowed| {
@@ -170,6 +175,7 @@ fn preset(name: &str) -> impl Iterator<Item = &'static str> {
         "llm_api_routes" => &[&OPENAI_ROUTES, &ANTHROPIC_ROUTES],
         "openai_routes" => &[&OPENAI_ROUTES],
         "anthropic_routes" => &[&ANTHROPIC_ROUTES],
+        "info_routes" => &[&INFO_ROUTES],
         _ => &[],
     };
     routes.iter().flat_map(|group| group.iter().copied())
@@ -570,6 +576,12 @@ mod tests {
     #[case::llm_api_routes_not_mcp(&["llm_api_routes"], "/mcp/sse", false)]
     #[case::openai_routes_chat(&["openai_routes"], "/v1/chat/completions", true)]
     #[case::openai_routes_responses(&["openai_routes"], "/v1/responses", true)]
+    #[case::openai_routes_models(&["openai_routes"], "/v1/models", true)]
+    #[case::openai_routes_not_model_retrieval(&["openai_routes"], "/v1/models/{model_id}", false)]
+    #[case::llm_api_routes_models(&["llm_api_routes"], "/models", true)]
+    #[case::info_routes_models(&["info_routes"], "/v1/models", true)]
+    #[case::info_routes_not_chat(&["info_routes"], "/v1/chat/completions", false)]
+    #[case::exact_models_allows_retrieval(&["/v1/models"], "/v1/models/{model_id}", true)]
     #[case::openai_routes_not_messages(&["openai_routes"], "/v1/messages", false)]
     #[case::anthropic_routes_messages(&["anthropic_routes"], "/v1/messages", true)]
     #[case::anthropic_routes_not_chat(&["anthropic_routes"], "/chat/completions", false)]

litellm-rust/crates/gateway-inference/Cargo.toml

@@ -11,6 +11,7 @@ axum = { workspace = true, features = ["json", "original-uri"] }
 base64.workspace = true
 bytes.workspace = true
 litellm-auth.workspace = true
+litellm-core-utils.workspace = true
 litellm-gateway-auth.workspace = true
 litellm-gateway-hooks.workspace = true
 litellm-inference.workspace = true

litellm-rust/crates/gateway-inference/src/error.rs

@@ -29,6 +29,9 @@ pub enum Error {
         "/v1/messages: Invalid model name passed in model={0}. Call `/v1/models` to view available models for your key."
     )]
     UnknownModel(String),
+    /// `/v1/models/{model_id}` for a model that does not exist or that the caller may not use.
+    #[error("The model `{0}` does not exist or is not accessible")]
+    ModelNotFound(String),
     #[error(transparent)]
     Route(#[from] RouteError),
     #[error(transparent)]
@@ -107,6 +110,7 @@ impl Error {
             | Self::UnknownModel(_)
             | Self::MissingParam { .. }
             | Self::InvalidMessages(_) => StatusCode::BAD_REQUEST,
+            Self::ModelNotFound(_) => StatusCode::NOT_FOUND,
             Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => {
                 StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY)
             }
@@ -122,6 +126,10 @@ impl Error {
         match self {
             Self::Auth(error) => error.into_response(),
             Self::Refused(refusal) => refused(refusal),
+            // The proxy raises a bare `HTTPException`, which FastAPI answers as `{"detail"}`.
+            Self::ModelNotFound(_) => {
+                (self.status(), Json(json!({ "detail": self.to_string() }))).into_response()
+            }
             other => (other.status(), Json(other.openai_body())).into_response(),
         }
     }

litellm-rust/crates/gateway-inference/src/lib.rs

@@ -9,12 +9,16 @@ mod client_side;
 mod error;
 mod hooks;
 pub mod messages;
+mod models;
 pub mod observe;
 mod request;
 
 use std::sync::{Arc, OnceLock};
 
-use axum::{Router, routing::post};
+use axum::{
+    Router,
+    routing::{get, post},
+};
 use litellm_http::{
     ClientVariant, HttpClientConfig,
     media::{MediaFetcher, UrlPolicy},
@@ -144,6 +148,10 @@ pub fn router(gateway: Arc<Gateway>) -> Router {
         .route("/v1/messages", post(messages::create))
         .route("/chat/completions", post(chat_completions::create))
         .route("/v1/chat/completions", post(chat_completions::create))
+        .route("/models", get(models::list))
+        .route("/v1/models", get(models::list))
+        .route("/models/{model_id}", get(models::retrieve))
+        .route("/v1/models/{model_id}", get(models::retrieve))
         .nest("/engines/{model}", model_routes())
         .nest("/openai/deployments/{model}", model_routes())
         // Not served: these routes skip the hooks (budgets, rate limits, spend), so a key could

litellm-rust/crates/gateway-inference/src/models.rs

@@ -0,0 +1,153 @@
+//! `GET /v1/models` and `GET /v1/models/{model_id}`: the model groups a caller may use, in the
+//! OpenAI shape the proxy's `model_list` and `model_info` answer with.
+
+use std::sync::Arc;
+
+use axum::{
+    Json,
+    extract::{Path, State},
+};
+use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
+use litellm_gateway_auth::{AuthenticatedRequest, AuthenticationMethod};
+use litellm_router::{Deployment, ModelGroup};
+use serde::Serialize;
+
+use crate::{Error, Gateway, request};
+
+/// The proxy's `DEFAULT_MODEL_CREATED_AT_TIME`.
+const CREATED: u64 = 1_677_610_602;
+
+#[derive(Debug, PartialEq, Serialize)]
+pub(crate) struct Model {
+    id: String,
+    object: &'static str,
+    created: u64,
+    owned_by: String,
+}
+
+impl Model {
+    fn new(id: &str, owned_by: &str) -> Self {
+        Self {
+            id: id.to_owned(),
+            object: "model",
+            created: CREATED,
+            owned_by: owned_by.to_owned(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub(crate) struct ModelList {
+    data: Vec<Model>,
+    object: &'static str,
+}
+
+/// One entry per model group the caller may call, in config order. The listing names every
+/// model's owner `openai`, as the proxy's does.
+pub(crate) async fn list(
+    State(gateway): State<Arc<Gateway>>,
+    identity: AuthenticatedRequest,
+) -> Json<ModelList> {
+    let mut data = Vec::new();
+    for group in gateway.models.groups() {
+        if visible(&identity, group, true).await.is_some() {
+            data.push(Model::new(group.name(), "openai"));
+        }
+    }
+    Json(ModelList {
+        data,
+        object: "list",
+    })
+}
+
+/// The model `model_id` names when the caller may call it, owned by its deployment's provider.
+pub(crate) async fn retrieve(
+    State(gateway): State<Arc<Gateway>>,
+    identity: AuthenticatedRequest,
+    Path(model_id): Path<String>,
+) -> Result<Json<Model>, Error> {
+    let Some(group) = gateway.models.group(&model_id) else {
+        return Err(Error::ModelNotFound(model_id));
+    };
+    match visible(&identity, group, false).await {
+        Some(deployment) => Ok(Json(Model::new(&model_id, provider(deployment)))),
+        None => Err(Error::ModelNotFound(model_id)),
+    }
+}
+
+/// The first deployment of `group` the caller may be routed to and is authorized for, as a
+/// request naming the group would be. When `listing`, only the master key sees a group every
+/// such deployment marks `discoverable: false`; any caller allowed it still retrieves it by
+/// name, as with the proxy. A wildcard group is never shown: the gateway routes model names exactly, so no
+/// request reaches it under another name.
+async fn visible<'a>(
+    identity: &AuthenticatedRequest,
+    group: &'a ModelGroup,
+    listing: bool,
+) -> Option<&'a Deployment> {
+    if group.name().contains('*') {
+        return None;
+    }
+    let hide_undiscoverable =
+        listing && identity.caller().authentication().method != AuthenticationMethod::MasterKey;
+    for deployment in group.deployments() {
+        if !identity.may_use_deployment(deployment.team_id.as_deref())
+            || (deployment.undiscoverable && hide_undiscoverable)
+        {
+            continue;
+        }
+        if request::authorize(identity, group.name(), deployment)
+            .await
+            .is_ok()
+        {
+            return Some(deployment);
+        }
+    }
+    None
+}
+
+/// The deployment's provider: its `custom_llm_provider`, else its model's prefix, else
+/// `openai`, which the proxy infers for a bare model name.
+fn provider(deployment: &Deployment) -> &str {
+    get_custom_llm_provider(&deployment.model, deployment.custom_llm_provider.as_deref())
+        .map_or("openai", |resolved| resolved.custom_llm_provider)
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::prefix("anthropic/claude-x", None, "anthropic")]
+    #[case::custom_provider("claude-x", Some("bedrock"), "bedrock")]
+    #[case::custom_provider_wins("openai/gpt-x", Some("azure"), "azure")]
+    #[case::bare_name("gpt-x", None, "openai")]
+    fn names_the_deployments_provider(
+        #[case] model: &str,
+        #[case] custom_llm_provider: Option<&str>,
+        #[case] expected: &str,
+    ) {
+        let deployment = Deployment {
+            model: model.to_owned(),
+            custom_llm_provider: custom_llm_provider.map(str::to_owned),
+            ..Deployment::default()
+        };
+
+        assert_eq!(provider(&deployment), expected);
+    }
+
+    #[test]
+    fn serializes_the_openai_model_shape() {
+        assert_eq!(
+            serde_json::to_value(Model::new("claude-a", "openai")).unwrap(),
+            serde_json::json!({
+                "id": "claude-a",
+                "object": "model",
+                "created": 1_677_610_602,
+                "owned_by": "openai",
+            })
+        );
+    }
+}

litellm-rust/crates/gateway/tests/virtual_keys.rs

@@ -187,6 +187,17 @@ async fn app() -> Router {
       api_key: sk-upstream
     model_info:
       team_id: team-2
+  - model_name: claude-hidden
+    litellm_params:
+      model: anthropic/claude-x
+      api_base: {upstream}
+      api_key: sk-upstream
+    model_info:
+      discoverable: false
+  - model_name: openai/*
+    litellm_params:
+      model: openai/*
+      api_key: sk-upstream
 router_settings:
   num_retries: 0
 general_settings:
@@ -284,6 +295,109 @@ async fn messages_accept_virtual_keys_from_every_key_header(
     }
 }
 
+async fn get(app: Router, path: &str, key: Option<&str>) -> (u16, Value) {
+    let request = Request::get(path);
+    let request = match key {
+        Some(key) => request.header("authorization", format!("Bearer {key}")),
+        None => request,
+    };
+    let response = app
+        .oneshot(request.body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+    let status = response.status().as_u16();
+    let body = to_bytes(response.into_body(), 1 << 20).await.unwrap();
+    (status, serde_json::from_slice(&body).unwrap())
+}
+
+/// The OpenAI model list of `ids`, as the proxy's `/v1/models` answers.
+fn model_list(ids: &[&str]) -> Value {
+    json!({
+        "data": ids
+            .iter()
+            .map(|id| json!({"id": id, "object": "model", "created": 1_677_610_602, "owned_by": "openai"}))
+            .collect::<Vec<_>>(),
+        "object": "list",
+    })
+}
+
+/// One entry per model group the key may call, in config order: team-owned groups only for
+/// their team, `discoverable: false` groups only for the master key, and no wildcard group.
+#[rstest]
+#[case::master_key("sk-master", &["claude-a", "claude-owned", "claude-mixed", "claude-hidden"])]
+#[case::all_models("sk-all", &["claude-a", "claude-mixed"])]
+#[case::key_models("sk-scoped", &["claude-a"])]
+#[case::key_model_not_deployed("sk-other", &[])]
+#[case::own_team("sk-team-2", &["claude-a", "claude-owned", "claude-mixed"])]
+#[case::other_team("sk-team-1", &["claude-a", "claude-mixed"])]
+#[case::team_models("sk-team-narrowed", &[])]
+#[case::openai_routes("sk-chat-only", &["claude-a", "claude-mixed"])]
+#[case::llm_api_routes("sk-messages-only", &["claude-a", "claude-mixed"])]
+#[tokio::test]
+async fn models_lists_the_groups_a_key_may_call(
+    #[future] app: Router,
+    #[case] key: &str,
+    #[case] expected: &[&str],
+    #[values("/v1/models", "/models")] path: &str,
+) {
+    let (status, body) = get(app.await, path, Some(key)).await;
+
+    assert_eq!(status, 200, "{body}");
+    assert_eq!(body, model_list(expected));
+}
+
+#[rstest]
+#[case::missing(None, 401, "auth_error")]
+#[case::unknown(Some("sk-unknown"), 401, "token_not_found_in_db")]
+#[case::expired(Some("sk-expired"), 401, "expired_key")]
+#[tokio::test]
+async fn models_requires_a_valid_key(
+    #[future] app: Router,
+    #[case] key: Option<&str>,
+    #[case] status: u16,
+    #[case] error_type: &str,
+) {
+    let (actual, body) = get(app.await, "/v1/models", key).await;
+
+    assert_eq!(actual, status, "{body}");
+    assert_eq!(body["error"]["type"], error_type);
+}
+
+#[rstest]
+#[case::master_key("sk-master", "claude-owned", 200)]
+#[case::shared("sk-all", "claude-a", 200)]
+#[case::own_team("sk-team-2", "claude-owned", 200)]
+#[case::other_team("sk-team-1", "claude-owned", 404)]
+#[case::key_models_exclude("sk-scoped", "claude-mixed", 404)]
+#[case::team_models_exclude("sk-team-narrowed", "claude-a", 404)]
+#[case::undiscoverable_retrievable_by_name("sk-all", "claude-hidden", 200)]
+#[case::undiscoverable_still_needs_access("sk-scoped", "claude-hidden", 404)]
+#[case::unknown_model("sk-master", "gpt-unknown", 404)]
+#[case::route_preset_lists_only("sk-chat-only", "claude-a", 403)]
+#[tokio::test]
+async fn model_retrieval_answers_only_models_the_key_may_call(
+    #[future] app: Router,
+    #[case] key: &str,
+    #[case] model: &str,
+    #[case] status: u16,
+    #[values("/v1/models", "/models")] prefix: &str,
+) {
+    let (actual, body) = get(app.await, &format!("{prefix}/{model}"), Some(key)).await;
+
+    assert_eq!(actual, status, "{body}");
+    match status {
+        200 => assert_eq!(
+            body,
+            json!({"id": model, "object": "model", "created": 1_677_610_602, "owned_by": "anthropic"})
+        ),
+        404 => assert_eq!(
+            body,
+            json!({"detail": format!("The model `{model}` does not exist or is not accessible")})
+        ),
+        _ => assert_eq!(body["error"]["type"], "auth_error"),
+    }
+}
+
 /// A Bedrock-shaped upstream counting the requests it receives: Converse calls get a Converse
 /// reply and invoke calls an Anthropic message.
 async fn bedrock_upstream() -> (String, Arc<AtomicUsize>) {

litellm-rust/crates/router/src/deployment.rs

@@ -24,6 +24,9 @@ pub struct Deployment {
     /// The model a deployment serves under another name, from `litellm_params.base_model` or
     /// `model_info.base_model`, which Azure's param rules and prices follow.
     pub base_model: Option<String>,
+    /// `model_info.discoverable: false`: the deployment still serves requests, but only the
+    /// master key sees it in model listings.
+    pub undiscoverable: bool,
 }
 
 impl Deployment {
@@ -49,6 +52,7 @@ impl Deployment {
             shaping: MessagesShaping::default(),
             team_id: None,
             id: None,
+            undiscoverable: false,
             base_model: params
                 .extra
                 .get("base_model")

litellm-rust/crates/router/src/lib.rs

@@ -27,6 +27,8 @@ use health::Health;
 /// The model groups of a config and the health of each deployment in them.
 pub struct Router {
     groups: HashMap<String, ModelGroup>,
+    /// The groups' names in the order the config first names them.
+    names: Box<[Box<str>]>,
     /// Every deployment's shared-state id and health, in a fixed order.
     deployments: Box<[(Box<str>, Arc<Health>)]>,
     shared: OnceLock<Arc<dyn SharedHealth>>,
@@ -91,6 +93,7 @@ impl Router {
                             .base_model
                             .clone()
                             .or_else(|| info_base_model(model)),
+                        undiscoverable: undiscoverable(model),
                         ..deployment
                     },
                 ))
@@ -121,6 +124,11 @@ impl Router {
                 (name, group)
             })
             .collect();
+        let names = order
+            .iter()
+            .filter(|(_, index)| *index == 0)
+            .map(|(name, _)| name.as_str().into())
+            .collect();
         let deployments = order
             .into_iter()
             .map(|(name, index)| {
@@ -130,6 +138,7 @@ impl Router {
             .collect();
         Self {
             groups,
+            names,
             deployments,
             shared: OnceLock::new(),
             policy,
@@ -162,6 +171,11 @@ impl Router {
         self.groups.get(model_name)
     }
 
+    /// Every model group, in the order the config first names each.
+    pub fn groups(&self) -> impl ExactSizeIterator<Item = &ModelGroup> {
+        self.names.iter().map(|name| &self.groups[&**name])
+    }
+
     pub fn policy(&self) -> &RoutingPolicy {
         &self.policy
     }
@@ -367,6 +381,14 @@ fn owner_team(model: &Model) -> Option<String> {
     }
 }
 
+/// `model_info.discoverable: false`, which keeps the deployment out of model listings.
+fn undiscoverable(model: &Model) -> bool {
+    matches!(
+        model.model_info.get("discoverable"),
+        Some(litellm_config::Value::Bool(false))
+    )
+}
+
 fn info_base_model(model: &Model) -> Option<String> {
     match model.model_info.get("base_model")? {
         litellm_config::Value::String(name) if !name.is_empty() => Some(name.clone()),

litellm-rust/crates/router/tests/router.rs

@@ -118,6 +118,7 @@ fn programmatic_deployments_preserve_overrides_and_share_their_group() {
         team_id: None,
         id: None,
         base_model: None,
+        undiscoverable: false,
     };
     let router = Router::from_iter([
         ("public-model".into(), Deployment::default()),
@@ -161,6 +162,31 @@ fn configuration_keeps_every_deployment_of_a_model_name_in_order() {
             .collect::<Vec<_>>(),
         ["provider/a", "provider/b", "provider/c"]
     );
+    assert_eq!(
+        router
+            .groups()
+            .map(|group| group.name())
+            .collect::<Vec<_>>(),
+        ["shared", "other"]
+    );
+}
+
+#[rstest]
+#[case::unset("{}", false)]
+#[case::discoverable("{discoverable: true}", false)]
+#[case::undiscoverable("{discoverable: false}", true)]
+#[case::not_a_bool("{discoverable: \"false\"}", false)]
+fn configuration_reads_discoverable(#[case] model_info: &str, #[case] undiscoverable: bool) {
+    let config = Config::from_yaml(&format!(
+        "model_list:
+  - model_name: m
+    litellm_params: {{model: provider/a}}
+    model_info: {model_info}"
+    ))
+    .unwrap();
+    let router = Router::from_config(&config).unwrap();
+
+    assert_eq!(only(&router, "m").undiscoverable, undiscoverable);
 }
 
 #[rstest]