From 3e44f7311e204b55a148de6581816cd6789c725c Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:09:32 +0800 Subject: [PATCH 1/6] fix(router): skip model-info discovery when token limits already configured (#42657) --- litellm/router.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 7267f6eb3ba..e542a9fb33b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10185,6 +10185,22 @@ class Router: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable + @staticmethod + def _deployment_token_limits_configured(model_info: object) -> bool: + """Both operative per-request token limits are explicitly configured. + + Discovery only populates ``max_input_tokens`` / ``max_output_tokens`` + (and ``max_tokens``) from the upstream ``/v1/models`` card; when a + deployment already sets both, the call cannot add anything, so it is + skipped to avoid billing scale-to-zero backends (issue #42657). + """ + if not isinstance(model_info, Mapping): + return False + return ( + model_info.get("max_input_tokens") is not None + and model_info.get("max_output_tokens") is not None + ) + async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: """Refresh token limits advertised by configured OpenAI-compatible deployments.""" deployments: Final = iter(tuple(self.model_list)) @@ -10219,6 +10235,13 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return + # Skip the /v1/models discovery call when the deployment already + # declares its token limits: discovery can only confirm values that + # are already configured, so the request is pure cost. On scale-to-zero + # backends (e.g. RunPod Serverless vLLM) each such call cold-starts a + # GPU worker and bills real money with no chat traffic (issue #42657). + if self._deployment_token_limits_configured(raw_deployment.get("model_info")): + return api_key: Final = params.api_key or dynamic_api_key headers: Final = TypeAdapter(Mapping[str, str]).validate_python( params.get("extra_headers") or params.get("headers") or MappingProxyType({}) From f14ae2fbfabfbad9b8b266a05a2bee029834c672 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:10:43 +0800 Subject: [PATCH 2/6] test(router): assert discovery is skipped for fully-configured deployments (#42657) --- .../test_router_model_cost_isolation.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index d73f5efa96b..92f45d52b39 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -384,6 +384,62 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa _invalidate_model_cost_lowercase_map() +async def test_discovery_skips_deployments_whose_limits_are_already_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Regression for #42657: token-limit discovery must not send GET /v1/models + # to a deployment that already declares both max_input_tokens and + # max_output_tokens. On scale-to-zero backends every such call cold-starts a + # worker and bills real money to confirm values the operator already set. + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + + router: Final = Router(model_list=[ + { + "model_name": "fully-configured", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://configured.test/v1", + "api_key": "local-key", + }, + "model_info": { + "id": "fully-configured", + "max_input_tokens": 2048, + "max_output_tokens": 1024, + }, + }, + { + "model_name": "needs-discovery", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://discover.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "needs-discovery"}, + }, + ]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + responder: Final = Mock(side_effect=respond) + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + + # Only the unconfigured deployment triggers a discovery request; the + # fully-configured one is skipped entirely. + discovery_hosts = {call.args[0].url.host for call in responder.call_args_list} + assert discovery_hosts == {"discover.test"} + assert responder.call_count == 1 + + # The configured deployment keeps its explicit limits; the discovered one + # picks up the upstream card. + assert router.get_configured_token_limits("fully-configured") == (2048, 1024) + assert router.get_configured_token_limits("needs-discovery") == (4096, 4096) + _invalidate_model_cost_lowercase_map() + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no From 512a29aa68d98f96be83cbc6209b8ff73068a44e Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:30:08 +0800 Subject: [PATCH 3/6] style(router): ruff format the discovery-skip helper (#42657) --- litellm/router.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index e542a9fb33b..46aa6ddc9f9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10196,10 +10196,7 @@ class Router: """ if not isinstance(model_info, Mapping): return False - return ( - model_info.get("max_input_tokens") is not None - and model_info.get("max_output_tokens") is not None - ) + return model_info.get("max_input_tokens") is not None and model_info.get("max_output_tokens") is not None async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: """Refresh token limits advertised by configured OpenAI-compatible deployments.""" From e0a10a3f009acc8434c659bd6f530ef4d613c556 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:38:20 +0800 Subject: [PATCH 4/6] test(router): add direct coverage for discovery-skip predicate (#42657) --- .../test_router_model_cost_isolation.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 92f45d52b39..5dc1cc21b9d 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -440,6 +440,19 @@ async def test_discovery_skips_deployments_whose_limits_are_already_configured( _invalidate_model_cost_lowercase_map() +def test_deployment_token_limits_configured_predicate() -> None: + # Direct coverage for the guard predicate #42657 adds: discovery is skipped + # only when BOTH per-request limits are explicitly set, so a partially + # configured deployment still discovers the missing one. + assert Router._deployment_token_limits_configured( + {"max_input_tokens": 2048, "max_output_tokens": 1024} + ) + assert not Router._deployment_token_limits_configured({"max_input_tokens": 2048}) + assert not Router._deployment_token_limits_configured({"max_output_tokens": 1024}) + assert not Router._deployment_token_limits_configured({}) + assert not Router._deployment_token_limits_configured(None) + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no From 92f6df45dbf3e0a01dee7f424e9954761acc7c44 Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 29 Sep 2026 17:11:53 +0800 Subject: [PATCH 5/6] style(router): drop redundant discovery-skip comment (#42657) The comment only restated the guard immediately below it and violated the repository comment policy (AGENTS.md); the helper docstring already explains why discovery is skipped when both token limits are configured. No behavior change. --- litellm/router.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 3c2d31acd26..bd3170c70d7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10252,11 +10252,6 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return - # Skip the /v1/models discovery call when the deployment already - # declares its token limits: discovery can only confirm values that - # are already configured, so the request is pure cost. On scale-to-zero - # backends (e.g. RunPod Serverless vLLM) each such call cold-starts a - # GPU worker and bills real money with no chat traffic (issue #42657). if self._deployment_token_limits_configured(raw_deployment.get("model_info")): return api_key: Final = params.api_key or dynamic_api_key From eee62875ae852c43cdef4bd6b13ef74e513bbf14 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:00:55 +0800 Subject: [PATCH 6/6] fix(types): keep the token-limit predicate's mapping access fully typed The new `_deployment_token_limits_configured` reads two keys off a bare `Mapping`, so both `.get` member accesses are only partially known and reportUnknownMemberType lands two over its ceiling on the basedpyright delta gate. Pin the value type with a cast before reading the keys. Checked with basedpyright 1.40.2 on litellm/router.py: reportUnknownMemberType 1530 -> 1528, reportExplicitAny unchanged at 28. --- litellm/router.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 13f30c6e021..826f0491fe8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10775,7 +10775,14 @@ class Router: """ if not isinstance(model_info, Mapping): return False - return model_info.get("max_input_tokens") is not None and model_info.get("max_output_tokens") is not None + # ``Mapping`` is unparameterised here, so pin the key/value types before + # reading the keys: bare ``Mapping`` makes both ``.get`` member accesses + # only partially known, which trips the basedpyright delta gate. + typed_model_info: Final = cast(Mapping[str, object], model_info) # cast-ok: isinstance check above + return ( + typed_model_info.get("max_input_tokens") is not None + and typed_model_info.get("max_output_tokens") is not None + ) async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: """Refresh token limits advertised by configured OpenAI-compatible deployments."""