diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 874dca4238f..f891c85320a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7545,9 +7545,17 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: """The widest limit any deployment behind the listed name declares for ``field``. A model group is normally one model behind several interchangeable deployments, so - there is a single value to report. When a group genuinely mixes models, reporting the - widest window keeps the listing independent of config order and agreeing with - ``/model_group/info``, which aggregates the same way for the Admin UI. + there is a single value to report and the choice of aggregate does not arise. + + When a group genuinely mixes models no single number is right, and the widest is the + deliberate pick over the narrowest for two reasons. It is what ``/model_group/info`` + has long reported to the Admin UI, so the two surfaces agree; disagreeing is the very + complaint this resolution path exists to fix. And of the two ways to be wrong, + under-advertising is worse: a client that trusts a narrowed window silently refuses + prompts the group would have served, while an over-long prompt that reaches a smaller + deployment comes back as a legible context-length error -- and does not reach one at + all when ``enable_pre_call_checks`` is set, which filters deployments the prompt does + not fit. """ limits: Final = tuple( limit for limit in (_first_token_limit(candidates, field) for candidates in candidate_sets) if limit is not None diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 588fd98abd1..eba9a85feb6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7350,6 +7350,20 @@ def test_get_model_listing_info_carries_configured_limits(): assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) +def test_widest_configured_limit_ignores_absent_and_malformed_values(): + model_infos = ( + {"max_input_tokens": 32000}, + {}, + {"max_input_tokens": "not-a-number"}, + {"max_input_tokens": "128000"}, + {"max_output_tokens": 4096}, + ) + + assert litellm.Router._widest_configured_limit(model_infos, "max_input_tokens") == 128000 + assert litellm.Router._widest_configured_limit(model_infos, "max_output_tokens") == 4096 + assert litellm.Router._widest_configured_limit((), "max_input_tokens") is None + + def test_get_model_listing_info_dedupes_interchangeable_deployments(): """The ordinary group is N deployments of one model, so it yields exactly one key.""" router = litellm.Router(