diff --git a/litellm/router.py b/litellm/router.py index 3809a5d6555..e2a5cb54255 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12021,6 +12021,26 @@ class Router: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable + @staticmethod + def _deployment_token_limits_configured(model_info: object) -> bool: + """Both operative per-request token limits are explicitly configured. + + Discovery only populates ``max_input_tokens`` / ``max_output_tokens`` + (and ``max_tokens``) from the upstream ``/v1/models`` card; when a + deployment already sets both, the call cannot add anything, so it is + skipped to avoid billing scale-to-zero backends (issue #42657). + """ + if not isinstance(model_info, Mapping): + return False + # ``Mapping`` is unparameterised here, so pin the key/value types before + # reading the keys: bare ``Mapping`` makes both ``.get`` member accesses + # only partially known, which trips the basedpyright delta gate. + typed_model_info: Final = cast(Mapping[str, object], model_info) # cast-ok: isinstance check above + return ( + typed_model_info.get("max_input_tokens") is not None + and typed_model_info.get("max_output_tokens") is not None + ) + async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: """Refresh token limits advertised by configured OpenAI-compatible deployments.""" deployments: Final = iter(tuple(self.model_list)) @@ -12055,6 +12075,8 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return + if self._deployment_token_limits_configured(raw_deployment.get("model_info")): + return api_key: Final = resolve_api_key_secret_reference( params.api_key or dynamic_api_key, model=model, diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..d466e624899 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -456,6 +456,75 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa _invalidate_model_cost_lowercase_map() +async def test_discovery_skips_deployments_whose_limits_are_already_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Regression for #42657: token-limit discovery must not send GET /v1/models + # to a deployment that already declares both max_input_tokens and + # max_output_tokens. On scale-to-zero backends every such call cold-starts a + # worker and bills real money to confirm values the operator already set. + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + + router: Final = Router(model_list=[ + { + "model_name": "fully-configured", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://configured.test/v1", + "api_key": "local-key", + }, + "model_info": { + "id": "fully-configured", + "max_input_tokens": 2048, + "max_output_tokens": 1024, + }, + }, + { + "model_name": "needs-discovery", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://discover.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "needs-discovery"}, + }, + ]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + responder: Final = Mock(side_effect=respond) + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + + # Only the unconfigured deployment triggers a discovery request; the + # fully-configured one is skipped entirely. + discovery_hosts = {call.args[0].url.host for call in responder.call_args_list} + assert discovery_hosts == {"discover.test"} + assert responder.call_count == 1 + + # The configured deployment keeps its explicit limits; the discovered one + # picks up the upstream card. + assert router.get_configured_token_limits("fully-configured") == (2048, 1024) + assert router.get_configured_token_limits("needs-discovery") == (4096, 4096) + _invalidate_model_cost_lowercase_map() + + +def test_deployment_token_limits_configured_predicate() -> None: + # Direct coverage for the guard predicate #42657 adds: discovery is skipped + # only when BOTH per-request limits are explicitly set, so a partially + # configured deployment still discovers the missing one. + assert Router._deployment_token_limits_configured( + {"max_input_tokens": 2048, "max_output_tokens": 1024} + ) + assert not Router._deployment_token_limits_configured({"max_input_tokens": 2048}) + assert not Router._deployment_token_limits_configured({"max_output_tokens": 1024}) + assert not Router._deployment_token_limits_configured({}) + assert not Router._deployment_token_limits_configured(None) + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no