diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 84bc27ef0d4..2ad8a08b8c3 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -523,7 +523,7 @@ export LITELLM_PROXY_API_KEY=sk-... lite model-groups list [--format table|json] ``` -Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. This is also what `lite autoroute configure` uses internally to discover what it can offer you. +Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. Note this route needs management access; `lite autoroute configure` instead discovers models through `/v1/models`, so it works with a key scoped to just the AI API routes #### Configure the Auto-Router diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py index 603cea38f6f..237705564ff 100644 --- a/litellm/proxy/client/cli/commands/autoroute/config.py +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -15,41 +15,25 @@ class DiscoveredModel(BaseModel): name: str mode: str = "chat" - input_cost_per_token: float | None = None - output_cost_per_token: float | None = None -class _RawModelGroup(BaseModel): +class _RawModelListing(BaseModel): model_config = ConfigDict(extra="ignore") - model_group: str - # Optional: some real deployments return an explicit `"mode": null` for models that - # were registered without a mode (seen for embedding models like voyage-4-large). - # ModelGroupInfo's own "chat" default (litellm/types/router.py) only applies when the - # key is missing entirely, not when it's present as null, so this must tolerate None. - mode: str | None = "chat" - input_cost_per_token: float | None = None - output_cost_per_token: float | None = None + id: str + # /v1/models attaches "mode" (sourced from the cost map) only for models it can resolve; + # a model whose mode is unknown arrives without the field, so default it to chat rather + # than dropping it, which keeps it selectable as a routing target in the wizard. + mode: str = "chat" -_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(list[_RawModelGroup]) +_RAW_MODEL_LISTING_ADAPTER = TypeAdapter(list[_RawModelListing]) def parse_discovered_models(raw: list[JsonValue]) -> tuple[DiscoveredModel, ...]: - """Validate a raw `/model_group/info` response into typed models.""" - parsed = _RAW_MODEL_GROUPS_ADAPTER.validate_python(raw) - return tuple( - DiscoveredModel( - name=group.model_group, - # A null mode means the server genuinely doesn't know what this model does; - # "unknown" (rather than guessing "chat") keeps it out of both chat_models() - # and embedding_models() instead of risking a wrong-mode deployment. - mode=group.mode or "unknown", - input_cost_per_token=group.input_cost_per_token, - output_cost_per_token=group.output_cost_per_token, - ) - for group in parsed - ) + """Validate a raw `/v1/models` response into typed models.""" + parsed = _RAW_MODEL_LISTING_ADAPTER.validate_python(raw) + return tuple(DiscoveredModel(name=item.id, mode=item.mode) for item in parsed) def chat_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]: diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index 8ad87315fb9..d3fe458d9f0 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -111,12 +111,12 @@ def run_configure_wizard(ctx: click.Context) -> Path: api_key = ctx.obj["api_key"] client = Client(base_url=base_url, api_key=api_key) - raw_groups = client.model_groups.info() - if not isinstance(raw_groups, list): + raw_models = client.models.list() + if not isinstance(raw_models, list): raise click.ClickException( - f"Unexpected response from /model_group/info: expected a list, got {type(raw_groups).__name__}" + f"Unexpected response from /v1/models: expected a list, got {type(raw_models).__name__}" ) - discovered = parse_discovered_models(raw_groups) + discovered = parse_discovered_models(raw_models) chat_pool = chat_models(discovered) embedding_pool = embedding_models(discovered) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1dbc0ad1837..5b81d1f2da3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6168,6 +6168,9 @@ def create_model_info_response( if model_cost_info is not None: max_input_tokens = coerce_token_limit(model_cost_info.get("max_input_tokens")) max_output_tokens = coerce_token_limit(model_cost_info.get("max_output_tokens")) + mode = model_cost_info.get("mode") + if isinstance(mode, str): + base["mode"] = mode if llm_router is not None: configured_input, configured_output = llm_router.get_configured_token_limits(model_id) diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index c3330da0d66..b59c0f2cf19 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -10,12 +10,16 @@ class ModelInfoMetadata(TypedDict): class ModelInfoResponse(TypedDict): - """OpenAI-compatible model object. `metadata` is present only when the - endpoint is called with include_metadata=true. + """OpenAI-compatible model object. `mode`, `max_input_tokens`, and + `max_output_tokens` are attached when the cost map knows them; `metadata` + is present only when the endpoint is called with include_metadata=true. """ id: str object: Literal["model"] created: int owned_by: str + mode: NotRequired[str] + max_input_tokens: NotRequired[int] + max_output_tokens: NotRequired[int] metadata: NotRequired[ModelInfoMetadata] diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py index 6ab484d5004..f399b6f957a 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py @@ -48,27 +48,22 @@ def _base_config(**overrides: Any) -> AutorouteConfig: class TestParseDiscoveredModels: def test_parses_valid_raw_list_into_typed_tuple(self): raw = [ - { - "model_group": "gpt-4o", - "mode": "chat", - "input_cost_per_token": 0.01, - "output_cost_per_token": 0.02, - }, - {"model_group": "text-embedding-3-small", "mode": "embedding"}, + {"id": "gpt-4o", "object": "model", "mode": "chat"}, + {"id": "text-embedding-3-small", "object": "model", "mode": "embedding"}, ] result = parse_discovered_models(raw) assert result == ( - DiscoveredModel(name="gpt-4o", mode="chat", input_cost_per_token=0.01, output_cost_per_token=0.02), + DiscoveredModel(name="gpt-4o", mode="chat"), DiscoveredModel(name="text-embedding-3-small", mode="embedding"), ) def test_ignores_unknown_extra_fields(self): - raw = [{"model_group": "gpt-4o", "mode": "chat", "totally_unknown_field": "whatever"}] + raw = [{"id": "gpt-4o", "mode": "chat", "created": 123, "owned_by": "openai", "max_input_tokens": 128000}] result = parse_discovered_models(raw) assert result == (DiscoveredModel(name="gpt-4o", mode="chat"),) def test_missing_mode_defaults_to_chat(self): - raw = [{"model_group": "gpt-4o"}] + raw = [{"id": "gpt-4o", "object": "model"}] result = parse_discovered_models(raw) assert result[0].mode == "chat" diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py index 78d4bd20338..a17fed36f52 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py @@ -16,22 +16,22 @@ from litellm.proxy.client.cli.commands.autoroute.config import DiscoveredModel from litellm.proxy.client.cli.commands.autoroute.wizard import run_configure_wizard CHAT_AND_EMBEDDING_GROUPS: List[Dict[str, Any]] = [ - {"model_group": "gpt-4o-mini", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, - {"model_group": "gpt-4o", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, - {"model_group": "claude-opus", "mode": "chat"}, - {"model_group": "o1", "mode": "chat"}, - {"model_group": "text-embedding-3-small", "mode": "embedding"}, + {"id": "gpt-4o-mini", "object": "model", "mode": "chat", "max_input_tokens": 128000}, + {"id": "gpt-4o", "object": "model", "mode": "chat", "max_input_tokens": 128000}, + {"id": "claude-opus", "object": "model", "mode": "chat"}, + {"id": "o1", "object": "model", "mode": "chat"}, + {"id": "text-embedding-3-small", "object": "model", "mode": "embedding"}, ] CHAT_ONLY_GROUPS: List[Dict[str, Any]] = [ - {"model_group": "gpt-4o-mini", "mode": "chat"}, - {"model_group": "gpt-4o", "mode": "chat"}, - {"model_group": "claude-opus", "mode": "chat"}, - {"model_group": "o1", "mode": "chat"}, + {"id": "gpt-4o-mini", "object": "model", "mode": "chat"}, + {"id": "gpt-4o", "object": "model", "mode": "chat"}, + {"id": "claude-opus", "object": "model", "mode": "chat"}, + {"id": "o1", "object": "model", "mode": "chat"}, ] EMBEDDING_ONLY_GROUPS: List[Dict[str, Any]] = [ - {"model_group": "text-embedding-3-small", "mode": "embedding"}, + {"id": "text-embedding-3-small", "object": "model", "mode": "embedding"}, ] @@ -73,7 +73,7 @@ def _run( patch.object(wizard_module, "_render_and_prompt_for_models", side_effect=_fake_prompt_for_models), patch.object(wizard_module, "_render_and_prompt_for_model", side_effect=_fake_prompt_for_model), ): - mock_client_cls.return_value.model_groups.info.return_value = raw_groups + mock_client_cls.return_value.models.list.return_value = raw_groups result = runner.invoke( _invoke_wizard, obj={"base_url": "http://localhost:4000", "api_key": "sk-test"}, @@ -262,7 +262,7 @@ class TestRunConfigureWizardNoChatModels: assert result.exit_code != 0 assert result.exception is None or not isinstance(result.exception, AssertionError) - assert "Unexpected response from /model_group/info" in result.output + assert "Unexpected response from /v1/models" in result.output assert not config_path.exists() @@ -275,7 +275,7 @@ class TestRunConfigureWizardNotInteractive: patch.object(wizard_module, "CONFIG_PATH", config_path), patch.object(wizard_module, "_is_interactive", return_value=False), ): - mock_client_cls.return_value.model_groups.info.return_value = CHAT_AND_EMBEDDING_GROUPS + mock_client_cls.return_value.models.list.return_value = CHAT_AND_EMBEDDING_GROUPS result = runner.invoke(_invoke_wizard, obj={"base_url": "http://localhost:4000", "api_key": "sk-test"}) assert result.exit_code != 0 diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 5ace46fc775..4673807a135 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -709,6 +709,39 @@ def test_create_model_info_response_reads_real_cost_map(): assert response["max_output_tokens"] > 0 +def test_create_model_info_response_includes_mode_from_lookup(): + response = create_model_info_response( + model_id="text-embedding-3-small", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info(mode="embedding"), + ) + + assert response["mode"] == "embedding" + + +def test_create_model_info_response_omits_mode_when_lookup_raises(): + response = create_model_info_response( + model_id="my-custom-deployment", + provider="openai", + llm_router=None, + get_model_info=_raise_unmapped, + ) + + assert "mode" not in response + + +def test_create_model_info_response_omits_non_string_mode(): + response = create_model_info_response( + model_id="some-model", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info(mode=None), + ) + + assert "mode" not in response + + class TestPostCallFailureHookLLMExceptionAlerting: """The llm_exceptions alert is for infra / LLM-API failures, not user errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized