From 44b3de5a83586eb09f5541097009720eb708fad9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:04:18 -0700 Subject: [PATCH] fix(proxy): source /v1/models token limits from the cost map instead of Router.get_model_group_info (#33721) * fix(proxy): source /v1/models token limits from cost map instead of Router.get_model_group_info Resolves the per-model get_model_group_info fan-out on GET /v1/models (and /models) that pegged the event loop on wildcard listings (#33636). create_model_info_response now reads max_input_tokens/max_output_tokens from litellm.get_model_info (the static cost map) rather than the router, which aggregated and deepcopied every deployment in a group per listed model. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): inject model-info lookup into create_model_info_response for deterministic coverage Inject the cost-map lookup (defaulting to litellm.get_model_info) so the except and max_output_tokens branches are exercised deterministically and the token-limit tests no longer hardcode mutable cost-map values. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(proxy): surface custom deployment token limits on /v1/models via cheap index lookup Add Router.get_configured_token_limits, an O(1) model-name index lookup that reads a concrete deployment's configured max_input_tokens/max_output_tokens without triggering pattern matching or deep copies. create_model_info_response layers this over the cost map so custom deployments absent from the cost map still surface their limits, and admin-configured limits override cost-map defaults, while wildcard-expanded names stay on the fast path. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 8536e3b80eabf0e0bed9f0f4eda18936137ebad2) --- litellm/proxy/utils.py | 52 +++-- litellm/router.py | 21 ++ tests/test_litellm/proxy/test_proxy_utils.py | 186 ++++++++++-------- .../proxy/utils/helpers/test_model_access.py | 12 +- tests/test_litellm/test_router.py | 47 +++++ 5 files changed, 209 insertions(+), 109 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d7649b524aa..50c69668871 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -19,6 +19,7 @@ from typing import ( Any, AsyncGenerator, Awaitable, + Callable, ClassVar, Dict, List, @@ -49,7 +50,7 @@ from litellm.proxy._types import ( from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import CallTypes, CallTypesLiteral +from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo try: from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( @@ -6098,6 +6099,7 @@ def create_model_info_response( include_metadata: bool = False, fallback_type: Optional[str] = None, llm_router: Optional["Router"] = None, + get_model_info: Callable[[str], ModelInfo] = litellm.get_model_info, ) -> ModelInfoResponse: """ Create a standardized OpenAI-compatible model object. @@ -6115,25 +6117,37 @@ def create_model_info_response( "owned_by": provider, } - # Surface context-window limits for OpenAI-compatible discovery clients. - # Only emitted when known, so wildcard routes and limitless backends stay clean. - # Limits are best-effort enrichment, so a single malformed deployment degrades - # to the base response rather than 500-ing the whole listing. + try: + model_cost_info: ModelInfo | None = get_model_info(model_id) + except Exception as e: + verbose_proxy_logger.debug( + "create_model_info_response: cost map lookup failed for %s: %s", + model_id, + e, + ) + model_cost_info = None + + max_input_tokens: int | None = None + max_output_tokens: int | None = None + if model_cost_info is not None: + cost_map_input = model_cost_info.get("max_input_tokens") + if cost_map_input is not None: + max_input_tokens = int(cost_map_input) + cost_map_output = model_cost_info.get("max_output_tokens") + if cost_map_output is not None: + max_output_tokens = int(cost_map_output) + if llm_router is not None: - try: - model_group_info = llm_router.get_model_group_info(model_id) - except Exception as e: - verbose_proxy_logger.debug( - "create_model_info_response: get_model_group_info failed for %s: %s", - model_id, - e, - ) - model_group_info = None - if model_group_info is not None: - if model_group_info.max_input_tokens is not None: - base["max_input_tokens"] = int(model_group_info.max_input_tokens) - if model_group_info.max_output_tokens is not None: - base["max_output_tokens"] = int(model_group_info.max_output_tokens) + configured_input, configured_output = llm_router.get_configured_token_limits(model_id) + if configured_input is not None: + max_input_tokens = configured_input + if configured_output is not None: + max_output_tokens = configured_output + + if max_input_tokens is not None: + base["max_input_tokens"] = max_input_tokens + if max_output_tokens is not None: + base["max_output_tokens"] = max_output_tokens if not include_metadata: return base diff --git a/litellm/router.py b/litellm/router.py index 6e773a06c7f..ced6090f82a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8357,6 +8357,27 @@ class Router: raise Exception("Model Name invalid - {}".format(type(model))) return None + def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": + """ + Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete + deployment's model_info for model_name, via O(1) index lookup. + + Returns (None, None) for wildcard-expanded or unknown names. Unlike + get_model_group_info, this never triggers pattern matching or deep copies, so it + is safe to call per listed model on the /v1/models hot path. + """ + deployment = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return (None, None) + + model_info = deployment.model_info + max_input = model_info.get("max_input_tokens") + max_output = model_info.get("max_output_tokens") + return ( + int(max_input) if max_input is not None else None, + int(max_output) if max_output is not None else None, + ) + def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Dict[str, Any]]: """ Get API credentials and provider info from a model name in model_list. diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index a909c510581..d2bdb1764a4 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -476,101 +476,118 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend: assert "response_cost" not in request_data +from typing import cast + from litellm.proxy.utils import create_model_info_response -from litellm.types.router import ModelGroupInfo +from litellm.types.utils import ModelInfo -def _router_returning(model_group_info): - router = MagicMock() - router.get_model_group_info = MagicMock(return_value=model_group_info) - return router +def _fake_model_info(**fields: int) -> ModelInfo: + return cast(ModelInfo, dict(fields)) -def test_create_model_info_response_includes_max_tokens_when_available(): - router = _router_returning( - ModelGroupInfo( - model_group="qwen-vllm", - providers=["hosted_vllm"], - max_input_tokens=32768, - max_output_tokens=8192, - ) +def _raise_unmapped(model_id: str) -> ModelInfo: + raise ValueError(f"This model isn't mapped yet: {model_id}") + + +def test_create_model_info_response_includes_max_tokens_from_lookup(): + response = create_model_info_response( + model_id="some-model", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info( + max_input_tokens=128000, max_output_tokens=16384 + ), ) + assert response["id"] == "some-model" + assert response["object"] == "model" + assert response["max_input_tokens"] == 128000 + assert response["max_output_tokens"] == 16384 + + +def test_create_model_info_response_does_not_call_router_group_info(): + router = MagicMock() + router.get_configured_token_limits.return_value = (None, None) + response = create_model_info_response( - model_id="qwen-vllm", provider="openai", llm_router=router + model_id="some-model", + provider="openai", + llm_router=router, + get_model_info=lambda _model: _fake_model_info( + max_input_tokens=128000, max_output_tokens=16384 + ), ) - router.get_model_group_info.assert_called_once_with("qwen-vllm") - assert response["id"] == "qwen-vllm" - assert response["object"] == "model" - assert response["max_input_tokens"] == 32768 - assert response["max_output_tokens"] == 8192 + router.get_model_group_info.assert_not_called() + assert response["max_input_tokens"] == 128000 + + +def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(): + router = MagicMock() + router.get_configured_token_limits.return_value = (32000, 8000) + + response = create_model_info_response( + model_id="my-custom-deployment", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + router.get_model_group_info.assert_not_called() + assert response["max_input_tokens"] == 32000 + assert response["max_output_tokens"] == 8000 + + +def test_create_model_info_response_deployment_limits_override_cost_map(): + router = MagicMock() + router.get_configured_token_limits.return_value = (200000, None) + + response = create_model_info_response( + model_id="gpt-4o", + provider="openai", + llm_router=router, + get_model_info=lambda _model: _fake_model_info( + max_input_tokens=128000, max_output_tokens=16384 + ), + ) + + assert response["max_input_tokens"] == 200000 + assert response["max_output_tokens"] == 16384 def test_create_model_info_response_emits_integer_token_counts(): - # ModelGroupInfo types the limits as float; OpenAI-compatible clients expect - # plain integers, so the response must not leak 128000.0. - router = _router_returning( - ModelGroupInfo( - model_group="gpt-4o", - providers=["openai"], - max_input_tokens=128000.0, - max_output_tokens=16384.0, - ) - ) - response = create_model_info_response( - model_id="gpt-4o", provider="openai", llm_router=router + model_id="some-model", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info( + max_input_tokens=128000, max_output_tokens=16384 + ), ) - assert response["max_input_tokens"] == 128000 assert isinstance(response["max_input_tokens"], int) - assert response["max_output_tokens"] == 16384 assert isinstance(response["max_output_tokens"], int) def test_create_model_info_response_omits_unknown_individual_limit(): - router = _router_returning( - ModelGroupInfo( - model_group="partial", - providers=["openai"], - max_input_tokens=4096, - max_output_tokens=None, - ) - ) - response = create_model_info_response( - model_id="partial", provider="openai", llm_router=router + model_id="some-embedding", + provider="openai", + llm_router=None, + get_model_info=lambda _model: _fake_model_info(max_input_tokens=8191), ) - assert response["max_input_tokens"] == 4096 + assert response["max_input_tokens"] == 8191 assert "max_output_tokens" not in response -def test_create_model_info_response_omits_limits_when_both_none(): - router = _router_returning( - ModelGroupInfo( - model_group="no-limits", - providers=["openai"], - max_input_tokens=None, - max_output_tokens=None, - ) - ) - +def test_create_model_info_response_omits_limits_when_lookup_raises(): response = create_model_info_response( - model_id="no-limits", provider="openai", llm_router=router - ) - - assert "max_input_tokens" not in response - assert "max_output_tokens" not in response - - -def test_create_model_info_response_omits_limits_when_group_unknown(): - # Wildcard routes / access groups have no ModelGroupInfo. - router = _router_returning(None) - - response = create_model_info_response( - model_id="openai/*", provider="openai", llm_router=router + model_id="openai/*", + provider="openai", + llm_router=None, + get_model_info=_raise_unmapped, ) assert response["id"] == "openai/*" @@ -578,32 +595,33 @@ def test_create_model_info_response_omits_limits_when_group_unknown(): assert "max_output_tokens" not in response -def test_create_model_info_response_degrades_when_group_info_raises(): - # A malformed deployment must not turn the listing into a 500; the entry - # falls back to the base fields without limits. - router = MagicMock() - router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) - - response = create_model_info_response( - model_id="broken", provider="openai", llm_router=router - ) - - assert response["id"] == "broken" - assert "max_input_tokens" not in response - assert "max_output_tokens" not in response - - def test_create_model_info_response_no_router_keeps_base_fields(): response = create_model_info_response( - model_id="some-model", provider="openai", llm_router=None + model_id="totally-unknown-model-xyz", + provider="openai", + llm_router=None, + get_model_info=_raise_unmapped, ) assert response == { - "id": "some-model", + "id": "totally-unknown-model-xyz", "object": "model", "created": response["created"], "owned_by": "openai", } + + +def test_create_model_info_response_reads_real_cost_map(): + response = create_model_info_response( + model_id="gpt-4o", provider="openai", llm_router=None + ) + + assert isinstance(response["max_input_tokens"], int) + assert response["max_input_tokens"] > 0 + assert isinstance(response["max_output_tokens"], int) + assert response["max_output_tokens"] > 0 + + class TestPostCallFailureHookLLMExceptionAlerting: """The llm_exceptions alert is for infra / LLM-API failures, not user errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized diff --git a/tests/test_litellm/proxy/utils/helpers/test_model_access.py b/tests/test_litellm/proxy/utils/helpers/test_model_access.py index b8e4013c960..59268e1427b 100644 --- a/tests/test_litellm/proxy/utils/helpers/test_model_access.py +++ b/tests/test_litellm/proxy/utils/helpers/test_model_access.py @@ -103,18 +103,16 @@ def test_is_known_vector_store_index_error_path_no_registry(monkeypatch): def test_create_model_info_response_happy_path_no_metadata(): result = create_model_info_response(model_id="gpt-4o", provider="openai") - assert result == { - "id": "gpt-4o", - "object": "model", - "created": result["created"], - "owned_by": "openai", - } snapshot = { "id": result["id"], "object": result["object"], "owned_by": result["owned_by"], "created_is_int": isinstance(result["created"], int), "metadata_absent": "metadata" not in result, + "max_input_tokens_positive_int": isinstance(result["max_input_tokens"], int) + and result["max_input_tokens"] > 0, + "max_output_tokens_positive_int": isinstance(result["max_output_tokens"], int) + and result["max_output_tokens"] > 0, } assert snapshot == { "id": "gpt-4o", @@ -122,6 +120,8 @@ def test_create_model_info_response_happy_path_no_metadata(): "owned_by": "openai", "created_is_int": True, "metadata_absent": True, + "max_input_tokens_positive_int": True, + "max_output_tokens_positive_int": True, } diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9c4d83ff7ea..3fca5aa6901 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5304,3 +5304,50 @@ class TestRouterRequestTimeoutPropagation: ) == 60 ) + + +def test_get_configured_token_limits_reads_deployment_model_info(): + router = litellm.Router( + model_list=[ + { + "model_name": "my-custom-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": 32000, "max_output_tokens": 8000}, + } + ] + ) + + assert router.get_configured_token_limits("my-custom-model") == (32000, 8000) + + +def test_get_configured_token_limits_returns_none_for_unset_or_unknown(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-limits-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_configured_token_limits("no-limits-model") == (None, None) + assert router.get_configured_token_limits("not-a-real-model") == (None, None) + + +def test_get_configured_token_limits_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"max_input_tokens": 12345}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert router.get_configured_token_limits( + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" + ) == (None, None)