This commit is contained in:
RichardoMu 2026-10-29 18:55:29 +08:00 • committed by GitHub
commit 2b36cb0813
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 91 additions and 0 deletions

View file

@ -12021,6 +12021,26 @@ class Router:
return None
return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable
@staticmethod
def _deployment_token_limits_configured(model_info: object) -> bool:
"""Both operative per-request token limits are explicitly configured.
Discovery only populates ``max_input_tokens`` / ``max_output_tokens``
(and ``max_tokens``) from the upstream ``/v1/models`` card; when a
deployment already sets both, the call cannot add anything, so it is
skipped to avoid billing scale-to-zero backends (issue #42657).
"""
if not isinstance(model_info, Mapping):
return False
# ``Mapping`` is unparameterised here, so pin the key/value types before
# reading the keys: bare ``Mapping`` makes both ``.get`` member accesses
# only partially known, which trips the basedpyright delta gate.
typed_model_info: Final = cast(Mapping[str, object], model_info) # cast-ok: isinstance check above
return (
typed_model_info.get("max_input_tokens") is not None
and typed_model_info.get("max_output_tokens") is not None
)
async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None:
"""Refresh token limits advertised by configured OpenAI-compatible deployments."""
deployments: Final = iter(tuple(self.model_list))
@ -12055,6 +12075,8 @@ class Router:
return
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
return
if self._deployment_token_limits_configured(raw_deployment.get("model_info")):
return
api_key: Final = resolve_api_key_secret_reference(
params.api_key or dynamic_api_key,
model=model,

View file

@ -456,6 +456,75 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa
_invalidate_model_cost_lowercase_map()
async def test_discovery_skips_deployments_whose_limits_are_already_configured(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Regression for #42657: token-limit discovery must not send GET /v1/models
# to a deployment that already declares both max_input_tokens and
# max_output_tokens. On scale-to-zero backends every such call cold-starts a
# worker and bills real money to confirm values the operator already set.
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]})
router: Final = Router(model_list=[
{
"model_name": "fully-configured",
"litellm_params": {
"model": "hosted_vllm/local-model",
"api_base": "https://configured.test/v1",
"api_key": "local-key",
},
"model_info": {
"id": "fully-configured",
"max_input_tokens": 2048,
"max_output_tokens": 1024,
},
},
{
"model_name": "needs-discovery",
"litellm_params": {
"model": "hosted_vllm/local-model",
"api_base": "https://discover.test/v1",
"api_key": "local-key",
},
"model_info": {"id": "needs-discovery"},
},
])
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
responder: Final = Mock(side_effect=respond)
async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
handler.client = client
await router.arefresh_model_info(client=handler)
# Only the unconfigured deployment triggers a discovery request; the
# fully-configured one is skipped entirely.
discovery_hosts = {call.args[0].url.host for call in responder.call_args_list}
assert discovery_hosts == {"discover.test"}
assert responder.call_count == 1
# The configured deployment keeps its explicit limits; the discovered one
# picks up the upstream card.
assert router.get_configured_token_limits("fully-configured") == (2048, 1024)
assert router.get_configured_token_limits("needs-discovery") == (4096, 4096)
_invalidate_model_cost_lowercase_map()
def test_deployment_token_limits_configured_predicate() -> None:
# Direct coverage for the guard predicate #42657 adds: discovery is skipped
# only when BOTH per-request limits are explicitly set, so a partially
# configured deployment still discovers the missing one.
assert Router._deployment_token_limits_configured(
{"max_input_tokens": 2048, "max_output_tokens": 1024}
)
assert not Router._deployment_token_limits_configured({"max_input_tokens": 2048})
assert not Router._deployment_token_limits_configured({"max_output_tokens": 1024})
assert not Router._deployment_token_limits_configured({})
assert not Router._deployment_token_limits_configured(None)
def test_should_not_pollute_shared_key_with_zero_cost_pricing():
"""
When deployment A has input_cost_per_token=0 and deployment B has no