From ba28c59b164822bf138dad318d474afe6b8dd492 Mon Sep 17 00:00:00 2001 From: Armaan Sandhu <74664101+Ar-maan05@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:29:22 +0530 Subject: [PATCH] feat(proxy): surface max_input_tokens/max_output_tokens on /v1/models (#30272) * feat(proxy): surface max_input_tokens/max_output_tokens on /v1/models * fix(proxy): degrade /v1/models gracefully when model-group lookup fails --------- Co-authored-by: Sameer Kankute --- litellm/proxy/utils.py | 22 ++++ tests/test_litellm/proxy/test_proxy_utils.py | 128 +++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 7e225c6cd1c..0423ba7d3f5 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6334,6 +6334,28 @@ def create_model_info_response( "owned_by": provider, } + # Surface context-window limits for OpenAI-compatible discovery clients. + # Only emitted when known, so wildcard routes and limitless backends stay clean. + # Limits are best-effort enrichment, so a single malformed deployment degrades + # to the base response rather than 500-ing the whole listing. + if llm_router is not None: + try: + model_group_info = llm_router.get_model_group_info(model_id) + except Exception as e: + verbose_proxy_logger.debug( + "create_model_info_response: get_model_group_info failed for %s: %s", + model_id, + e, + ) + model_group_info = None + if model_group_info is not None: + if model_group_info.max_input_tokens is not None: + model_info["max_input_tokens"] = int(model_group_info.max_input_tokens) + if model_group_info.max_output_tokens is not None: + model_info["max_output_tokens"] = int( + model_group_info.max_output_tokens + ) + # Add metadata if requested if include_metadata: metadata = {} diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index d6b4c48a80c..3e86f0e8f3c 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -427,6 +427,134 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime: assert "litellm_logging_obj" not in request_data +from litellm.proxy.utils import create_model_info_response +from litellm.types.router import ModelGroupInfo + + +def _router_returning(model_group_info): + router = MagicMock() + router.get_model_group_info = MagicMock(return_value=model_group_info) + return router + + +def test_create_model_info_response_includes_max_tokens_when_available(): + router = _router_returning( + ModelGroupInfo( + model_group="qwen-vllm", + providers=["hosted_vllm"], + max_input_tokens=32768, + max_output_tokens=8192, + ) + ) + + response = create_model_info_response( + model_id="qwen-vllm", provider="openai", llm_router=router + ) + + router.get_model_group_info.assert_called_once_with("qwen-vllm") + assert response["id"] == "qwen-vllm" + assert response["object"] == "model" + assert response["max_input_tokens"] == 32768 + assert response["max_output_tokens"] == 8192 + + +def test_create_model_info_response_emits_integer_token_counts(): + # ModelGroupInfo types the limits as float; OpenAI-compatible clients expect + # plain integers, so the response must not leak 128000.0. + router = _router_returning( + ModelGroupInfo( + model_group="gpt-4o", + providers=["openai"], + max_input_tokens=128000.0, + max_output_tokens=16384.0, + ) + ) + + response = create_model_info_response( + model_id="gpt-4o", provider="openai", llm_router=router + ) + + assert response["max_input_tokens"] == 128000 + assert isinstance(response["max_input_tokens"], int) + assert response["max_output_tokens"] == 16384 + assert isinstance(response["max_output_tokens"], int) + + +def test_create_model_info_response_omits_unknown_individual_limit(): + router = _router_returning( + ModelGroupInfo( + model_group="partial", + providers=["openai"], + max_input_tokens=4096, + max_output_tokens=None, + ) + ) + + response = create_model_info_response( + model_id="partial", provider="openai", llm_router=router + ) + + assert response["max_input_tokens"] == 4096 + assert "max_output_tokens" not in response + + +def test_create_model_info_response_omits_limits_when_both_none(): + router = _router_returning( + ModelGroupInfo( + model_group="no-limits", + providers=["openai"], + max_input_tokens=None, + max_output_tokens=None, + ) + ) + + response = create_model_info_response( + model_id="no-limits", provider="openai", llm_router=router + ) + + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + +def test_create_model_info_response_omits_limits_when_group_unknown(): + # Wildcard routes / access groups have no ModelGroupInfo. + router = _router_returning(None) + + response = create_model_info_response( + model_id="openai/*", provider="openai", llm_router=router + ) + + assert response["id"] == "openai/*" + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + +def test_create_model_info_response_degrades_when_group_info_raises(): + # A malformed deployment must not turn the listing into a 500; the entry + # falls back to the base fields without limits. + router = MagicMock() + router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) + + response = create_model_info_response( + model_id="broken", provider="openai", llm_router=router + ) + + assert response["id"] == "broken" + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + +def test_create_model_info_response_no_router_keeps_base_fields(): + response = create_model_info_response( + model_id="some-model", provider="openai", llm_router=None + ) + + assert response == { + "id": "some-model", + "object": "model", + "created": response["created"], + "owned_by": "openai", + } class TestPostCallFailureHookLLMExceptionAlerting: """The llm_exceptions alert is for infra / LLM-API failures, not user errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized