fix(autoroute): discover models via /v1/models so an AI-API-only key works (#34259)

Co-authored-by: Krrish Dholakia <krrishdholakia@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/client/cli/README.md

@@ -523,7 +523,7 @@ export LITELLM_PROXY_API_KEY=sk-...
 lite model-groups list [--format table|json]
 ```
 
-Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. This is also what `lite autoroute configure` uses internally to discover what it can offer you.
+Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. Note this route needs management access; `lite autoroute configure` instead discovers models through `/v1/models`, so it works with a key scoped to just the AI API routes
 
 #### Configure the Auto-Router
 

litellm/proxy/client/cli/commands/autoroute/config.py

@@ -15,41 +15,25 @@ class DiscoveredModel(BaseModel):
 
     name: str
     mode: str = "chat"
-    input_cost_per_token: float | None = None
-    output_cost_per_token: float | None = None
 
 
-class _RawModelGroup(BaseModel):
+class _RawModelListing(BaseModel):
     model_config = ConfigDict(extra="ignore")
 
-    model_group: str
-    # Optional: some real deployments return an explicit `"mode": null` for models that
-    # were registered without a mode (seen for embedding models like voyage-4-large).
-    # ModelGroupInfo's own "chat" default (litellm/types/router.py) only applies when the
-    # key is missing entirely, not when it's present as null, so this must tolerate None.
-    mode: str | None = "chat"
-    input_cost_per_token: float | None = None
-    output_cost_per_token: float | None = None
+    id: str
+    # /v1/models attaches "mode" (sourced from the cost map) only for models it can resolve;
+    # a model whose mode is unknown arrives without the field, so default it to chat rather
+    # than dropping it, which keeps it selectable as a routing target in the wizard.
+    mode: str = "chat"
 
 
-_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(list[_RawModelGroup])
+_RAW_MODEL_LISTING_ADAPTER = TypeAdapter(list[_RawModelListing])
 
 
 def parse_discovered_models(raw: list[JsonValue]) -> tuple[DiscoveredModel, ...]:
-    """Validate a raw `/model_group/info` response into typed models."""
-    parsed = _RAW_MODEL_GROUPS_ADAPTER.validate_python(raw)
-    return tuple(
-        DiscoveredModel(
-            name=group.model_group,
-            # A null mode means the server genuinely doesn't know what this model does;
-            # "unknown" (rather than guessing "chat") keeps it out of both chat_models()
-            # and embedding_models() instead of risking a wrong-mode deployment.
-            mode=group.mode or "unknown",
-            input_cost_per_token=group.input_cost_per_token,
-            output_cost_per_token=group.output_cost_per_token,
-        )
-        for group in parsed
-    )
+    """Validate a raw `/v1/models` response into typed models."""
+    parsed = _RAW_MODEL_LISTING_ADAPTER.validate_python(raw)
+    return tuple(DiscoveredModel(name=item.id, mode=item.mode) for item in parsed)
 
 
 def chat_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]:

litellm/proxy/client/cli/commands/autoroute/wizard.py

@@ -111,12 +111,12 @@ def run_configure_wizard(ctx: click.Context) -> Path:
     api_key = ctx.obj["api_key"]
     client = Client(base_url=base_url, api_key=api_key)
 
-    raw_groups = client.model_groups.info()
-    if not isinstance(raw_groups, list):
+    raw_models = client.models.list()
+    if not isinstance(raw_models, list):
         raise click.ClickException(
-            f"Unexpected response from /model_group/info: expected a list, got {type(raw_groups).__name__}"
+            f"Unexpected response from /v1/models: expected a list, got {type(raw_models).__name__}"
         )
-    discovered = parse_discovered_models(raw_groups)
+    discovered = parse_discovered_models(raw_models)
     chat_pool = chat_models(discovered)
     embedding_pool = embedding_models(discovered)
 

litellm/proxy/utils.py

@@ -6168,6 +6168,9 @@ def create_model_info_response(
     if model_cost_info is not None:
         max_input_tokens = coerce_token_limit(model_cost_info.get("max_input_tokens"))
         max_output_tokens = coerce_token_limit(model_cost_info.get("max_output_tokens"))
+        mode = model_cost_info.get("mode")
+        if isinstance(mode, str):
+            base["mode"] = mode
 
     if llm_router is not None:
         configured_input, configured_output = llm_router.get_configured_token_limits(model_id)

litellm/types/proxy/model_listing.py

@@ -10,12 +10,16 @@ class ModelInfoMetadata(TypedDict):
 
 
 class ModelInfoResponse(TypedDict):
-    """OpenAI-compatible model object. `metadata` is present only when the
-    endpoint is called with include_metadata=true.
+    """OpenAI-compatible model object. `mode`, `max_input_tokens`, and
+    `max_output_tokens` are attached when the cost map knows them; `metadata`
+    is present only when the endpoint is called with include_metadata=true.
     """
 
     id: str
     object: Literal["model"]
     created: int
     owned_by: str
+    mode: NotRequired[str]
+    max_input_tokens: NotRequired[int]
+    max_output_tokens: NotRequired[int]
     metadata: NotRequired[ModelInfoMetadata]

tests/test_litellm/proxy/client/cli/autoroute/test_config.py

@@ -48,27 +48,22 @@ def _base_config(**overrides: Any) -> AutorouteConfig:
 class TestParseDiscoveredModels:
     def test_parses_valid_raw_list_into_typed_tuple(self):
         raw = [
-            {
-                "model_group": "gpt-4o",
-                "mode": "chat",
-                "input_cost_per_token": 0.01,
-                "output_cost_per_token": 0.02,
-            },
-            {"model_group": "text-embedding-3-small", "mode": "embedding"},
+            {"id": "gpt-4o", "object": "model", "mode": "chat"},
+            {"id": "text-embedding-3-small", "object": "model", "mode": "embedding"},
         ]
         result = parse_discovered_models(raw)
         assert result == (
-            DiscoveredModel(name="gpt-4o", mode="chat", input_cost_per_token=0.01, output_cost_per_token=0.02),
+            DiscoveredModel(name="gpt-4o", mode="chat"),
             DiscoveredModel(name="text-embedding-3-small", mode="embedding"),
         )
 
     def test_ignores_unknown_extra_fields(self):
-        raw = [{"model_group": "gpt-4o", "mode": "chat", "totally_unknown_field": "whatever"}]
+        raw = [{"id": "gpt-4o", "mode": "chat", "created": 123, "owned_by": "openai", "max_input_tokens": 128000}]
         result = parse_discovered_models(raw)
         assert result == (DiscoveredModel(name="gpt-4o", mode="chat"),)
 
     def test_missing_mode_defaults_to_chat(self):
-        raw = [{"model_group": "gpt-4o"}]
+        raw = [{"id": "gpt-4o", "object": "model"}]
         result = parse_discovered_models(raw)
         assert result[0].mode == "chat"
 

tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py

@@ -16,22 +16,22 @@ from litellm.proxy.client.cli.commands.autoroute.config import DiscoveredModel
 from litellm.proxy.client.cli.commands.autoroute.wizard import run_configure_wizard
 
 CHAT_AND_EMBEDDING_GROUPS: List[Dict[str, Any]] = [
-    {"model_group": "gpt-4o-mini", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02},
-    {"model_group": "gpt-4o", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02},
-    {"model_group": "claude-opus", "mode": "chat"},
-    {"model_group": "o1", "mode": "chat"},
-    {"model_group": "text-embedding-3-small", "mode": "embedding"},
+    {"id": "gpt-4o-mini", "object": "model", "mode": "chat", "max_input_tokens": 128000},
+    {"id": "gpt-4o", "object": "model", "mode": "chat", "max_input_tokens": 128000},
+    {"id": "claude-opus", "object": "model", "mode": "chat"},
+    {"id": "o1", "object": "model", "mode": "chat"},
+    {"id": "text-embedding-3-small", "object": "model", "mode": "embedding"},
 ]
 
 CHAT_ONLY_GROUPS: List[Dict[str, Any]] = [
-    {"model_group": "gpt-4o-mini", "mode": "chat"},
-    {"model_group": "gpt-4o", "mode": "chat"},
-    {"model_group": "claude-opus", "mode": "chat"},
-    {"model_group": "o1", "mode": "chat"},
+    {"id": "gpt-4o-mini", "object": "model", "mode": "chat"},
+    {"id": "gpt-4o", "object": "model", "mode": "chat"},
+    {"id": "claude-opus", "object": "model", "mode": "chat"},
+    {"id": "o1", "object": "model", "mode": "chat"},
 ]
 
 EMBEDDING_ONLY_GROUPS: List[Dict[str, Any]] = [
-    {"model_group": "text-embedding-3-small", "mode": "embedding"},
+    {"id": "text-embedding-3-small", "object": "model", "mode": "embedding"},
 ]
 
 
@@ -73,7 +73,7 @@ def _run(
         patch.object(wizard_module, "_render_and_prompt_for_models", side_effect=_fake_prompt_for_models),
         patch.object(wizard_module, "_render_and_prompt_for_model", side_effect=_fake_prompt_for_model),
     ):
-        mock_client_cls.return_value.model_groups.info.return_value = raw_groups
+        mock_client_cls.return_value.models.list.return_value = raw_groups
         result = runner.invoke(
             _invoke_wizard,
             obj={"base_url": "http://localhost:4000", "api_key": "sk-test"},
@@ -262,7 +262,7 @@ class TestRunConfigureWizardNoChatModels:
 
         assert result.exit_code != 0
         assert result.exception is None or not isinstance(result.exception, AssertionError)
-        assert "Unexpected response from /model_group/info" in result.output
+        assert "Unexpected response from /v1/models" in result.output
         assert not config_path.exists()
 
 
@@ -275,7 +275,7 @@ class TestRunConfigureWizardNotInteractive:
             patch.object(wizard_module, "CONFIG_PATH", config_path),
             patch.object(wizard_module, "_is_interactive", return_value=False),
         ):
-            mock_client_cls.return_value.model_groups.info.return_value = CHAT_AND_EMBEDDING_GROUPS
+            mock_client_cls.return_value.models.list.return_value = CHAT_AND_EMBEDDING_GROUPS
             result = runner.invoke(_invoke_wizard, obj={"base_url": "http://localhost:4000", "api_key": "sk-test"})
 
         assert result.exit_code != 0

tests/test_litellm/proxy/test_proxy_utils.py

@@ -709,6 +709,39 @@ def test_create_model_info_response_reads_real_cost_map():
     assert response["max_output_tokens"] > 0
 
 
+def test_create_model_info_response_includes_mode_from_lookup():
+    response = create_model_info_response(
+        model_id="text-embedding-3-small",
+        provider="openai",
+        llm_router=None,
+        get_model_info=lambda _model: _fake_model_info(mode="embedding"),
+    )
+
+    assert response["mode"] == "embedding"
+
+
+def test_create_model_info_response_omits_mode_when_lookup_raises():
+    response = create_model_info_response(
+        model_id="my-custom-deployment",
+        provider="openai",
+        llm_router=None,
+        get_model_info=_raise_unmapped,
+    )
+
+    assert "mode" not in response
+
+
+def test_create_model_info_response_omits_non_string_mode():
+    response = create_model_info_response(
+        model_id="some-model",
+        provider="openai",
+        llm_router=None,
+        get_model_info=lambda _model: _fake_model_info(mode=None),
+    )
+
+    assert "mode" not in response
+
+
 class TestPostCallFailureHookLLMExceptionAlerting:
     """The llm_exceptions alert is for infra / LLM-API failures, not user
     errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized