From e3641ee685025463df98380e3504d11277f20a52 Mon Sep 17 00:00:00 2001 From: todayim <809634488@qq.com> Date: Wed, 5 Aug 2026 14:57:53 +0800 Subject: [PATCH] feat(proxy): surface deployment-advertised max_model_len in GET /v1/models OpenAI-compatible inference servers such as vLLM and SGLang report the per-request context length of a deployment under the field name max_model_len on their own /v1/models, and operators already register that value into LiteLLM deployment model_info. create_model_info_response now passes it through: it reads max_model_len from the cost map and from the concrete deployment's model_info (deployment wins), and includes it on /v1/models and /v1/models/{model_id} whenever it is known. Malformed config values are treated as absent via the shared coerce_token_limit helper, matching the existing max_input_tokens/max_output_tokens behavior, so the listing continues to degrade gracefully instead of failing. Adds Router.get_configured_max_model_len, an O(1) deployment index lookup with no pattern matching or deep copies, and declares the field on the cost-map ModelInfo schema and the ModelInfoResponse listing type. Tests cover cost-map surfacing, deployment override, omission when unset, wildcard non-expansion, numeric-string coercion, and malformed values. --- litellm/proxy/utils.py | 10 +++ litellm/router.py | 19 +++++ litellm/types/proxy/model_listing.py | 8 ++- litellm/types/utils.py | 3 + tests/test_litellm/proxy/test_proxy_utils.py | 62 ++++++++++++++++ tests/test_litellm/test_router.py | 76 ++++++++++++++++++++ 6 files changed, 175 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 99e566c2da1..e094f0077c6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6454,9 +6454,11 @@ def create_model_info_response( max_input_tokens: int | None = None max_output_tokens: int | None = None + max_model_len: int | None = None if model_cost_info is not None: max_input_tokens = coerce_token_limit(model_cost_info.get("max_input_tokens")) max_output_tokens = coerce_token_limit(model_cost_info.get("max_output_tokens")) + max_model_len = coerce_token_limit(model_cost_info.get("max_model_len")) mode: Final = model_cost_info.get("mode") if isinstance(mode, str): base["mode"] = mode @@ -6467,11 +6469,19 @@ def create_model_info_response( max_input_tokens = configured_input if configured_output is not None: max_output_tokens = configured_output + # Coerce here as well: callers with duck-typed routers (and mocks that + # auto-stub unknown methods) must not leak an arbitrary object into the + # response. get_configured_max_model_len already coerces real values. + configured_model_len: Final = coerce_token_limit(llm_router.get_configured_max_model_len(model_id)) + if configured_model_len is not None: + max_model_len = configured_model_len if max_input_tokens is not None: base["max_input_tokens"] = max_input_tokens if max_output_tokens is not None: base["max_output_tokens"] = max_output_tokens + if max_model_len is not None: + base["max_model_len"] = max_model_len if not include_metadata: return base diff --git a/litellm/router.py b/litellm/router.py index 9cde292657c..d5e16ebc4a2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8653,6 +8653,25 @@ class Router: coerce_token_limit(model_info.get("max_output_tokens")), ) + def get_configured_max_model_len(self, model_name: str) -> "int | None": + """ + Return max_model_len explicitly configured in a concrete deployment's + model_info for model_name, via O(1) index lookup. + + OpenAI-compatible inference servers such as vLLM and SGLang report the + per-request context length of a deployment under this name, so the field + uses their spelling rather than a LiteLLM-specific one. Returns None for + wildcard-expanded or unknown names, and treats a malformed configured + value as absent rather than failing the listing. Like + get_configured_token_limits, this never triggers pattern matching or deep + copies, so it is safe to call per listed model on the /v1/models hot path. + """ + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + return coerce_token_limit(deployment.model_info.get("max_model_len")) + def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> dict[str, Any] | None: diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index b59c0f2cf19..cc454c7b16f 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -10,9 +10,10 @@ class ModelInfoMetadata(TypedDict): class ModelInfoResponse(TypedDict): - """OpenAI-compatible model object. `mode`, `max_input_tokens`, and - `max_output_tokens` are attached when the cost map knows them; `metadata` - is present only when the endpoint is called with include_metadata=true. + """OpenAI-compatible model object. `mode`, `max_input_tokens`, + `max_output_tokens`, and `max_model_len` are attached when configured or + known to the cost map; `metadata` is present only when the endpoint is + called with include_metadata=true. """ id: str @@ -22,4 +23,5 @@ class ModelInfoResponse(TypedDict): mode: NotRequired[str] max_input_tokens: NotRequired[int] max_output_tokens: NotRequired[int] + max_model_len: NotRequired[int] metadata: NotRequired[ModelInfoMetadata] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 77f83c5b6f8..40c447ac7d9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -197,6 +197,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): max_tokens: Required[Optional[int]] max_input_tokens: Required[Optional[int]] max_output_tokens: Required[Optional[int]] + # per-request context length as advertised by OpenAI-compatible backends + # (vLLM and SGLang report it under this name on their own /v1/models) + max_model_len: Optional[int] input_cost_per_token: Required[Optional[float]] input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 3421751d962..e5914d160be 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -684,6 +684,68 @@ def test_create_model_info_response_omits_limits_when_lookup_raises(): assert "max_output_tokens" not in response +def test_create_model_info_response_includes_max_model_len_from_cost_map(): + response = create_model_info_response( + model_id="some-model", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info(max_model_len=131072), + ) + + assert response["max_model_len"] == 131072 + + +def test_create_model_info_response_omits_max_model_len_when_unknown(): + response = create_model_info_response( + model_id="some-model", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info(max_input_tokens=8191), + ) + + assert response["max_input_tokens"] == 8191 + assert "max_model_len" not in response + + +def test_create_model_info_response_deployment_max_model_len_overrides_cost_map(): + router = MagicMock() + router.get_configured_token_limits.return_value = (None, None) + router.get_configured_max_model_len.return_value = 65536 + + response = create_model_info_response( + model_id="some-model", + provider="openai", + llm_router=router, + get_model_info=lambda _model: _fake_model_info(max_model_len=131072), + ) + + assert response["max_model_len"] == 65536 + + +def test_create_model_info_response_survives_malformed_configured_max_model_len(): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "bad-ctx-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_model_len": "131,072"}, + } + ] + ) + + response = create_model_info_response( + model_id="bad-ctx-model", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert response["id"] == "bad-ctx-model" + assert "max_model_len" not in response + + def test_create_model_info_response_no_router_keeps_base_fields(): response = create_model_info_response( model_id="totally-unknown-model-xyz", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 33aab1cf708..21d5e9a19f6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6590,6 +6590,82 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +def test_get_configured_max_model_len_reads_deployment_model_info(): + router = litellm.Router( + model_list=[ + { + "model_name": "my-custom-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_model_len": 131072}, + } + ] + ) + + assert router.get_configured_max_model_len("my-custom-model") == 131072 + + +def test_get_configured_max_model_len_returns_none_for_unset_or_unknown(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-ctx-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_configured_max_model_len("no-ctx-model") is None + assert router.get_configured_max_model_len("not-a-real-model") is None + + +def test_get_configured_max_model_len_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"max_model_len": 12345}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert router.get_configured_max_model_len("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") is None + + +def test_get_configured_max_model_len_treats_malformed_values_as_absent(): + malformed = ["", "131,072", True, [131072], {"len": 131072}] + router = litellm.Router( + model_list=[ + { + "model_name": f"bad-ctx-{i}", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_model_len": bad}, + } + for i, bad in enumerate(malformed) + ] + ) + + for i in range(len(malformed)): + assert router.get_configured_max_model_len(f"bad-ctx-{i}") is None + + +def test_get_configured_max_model_len_coerces_numeric_strings(): + router = litellm.Router( + model_list=[ + { + "model_name": "quoted-ctx-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_model_len": "131072"}, + } + ] + ) + + assert router.get_configured_max_model_len("quoted-ctx-model") == 131072 + + @pytest.mark.asyncio async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): import httpx