mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 27846eb941 into d037d86a94
This commit is contained in:
commit
2b36cb0813
2 changed files with 91 additions and 0 deletions
|
|
@ -12021,6 +12021,26 @@ class Router:
|
|||
return None
|
||||
return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable
|
||||
|
||||
@staticmethod
|
||||
def _deployment_token_limits_configured(model_info: object) -> bool:
|
||||
"""Both operative per-request token limits are explicitly configured.
|
||||
|
||||
Discovery only populates ``max_input_tokens`` / ``max_output_tokens``
|
||||
(and ``max_tokens``) from the upstream ``/v1/models`` card; when a
|
||||
deployment already sets both, the call cannot add anything, so it is
|
||||
skipped to avoid billing scale-to-zero backends (issue #42657).
|
||||
"""
|
||||
if not isinstance(model_info, Mapping):
|
||||
return False
|
||||
# ``Mapping`` is unparameterised here, so pin the key/value types before
|
||||
# reading the keys: bare ``Mapping`` makes both ``.get`` member accesses
|
||||
# only partially known, which trips the basedpyright delta gate.
|
||||
typed_model_info: Final = cast(Mapping[str, object], model_info) # cast-ok: isinstance check above
|
||||
return (
|
||||
typed_model_info.get("max_input_tokens") is not None
|
||||
and typed_model_info.get("max_output_tokens") is not None
|
||||
)
|
||||
|
||||
async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None:
|
||||
"""Refresh token limits advertised by configured OpenAI-compatible deployments."""
|
||||
deployments: Final = iter(tuple(self.model_list))
|
||||
|
|
@ -12055,6 +12075,8 @@ class Router:
|
|||
return
|
||||
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
|
||||
return
|
||||
if self._deployment_token_limits_configured(raw_deployment.get("model_info")):
|
||||
return
|
||||
api_key: Final = resolve_api_key_secret_reference(
|
||||
params.api_key or dynamic_api_key,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -456,6 +456,75 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_discovery_skips_deployments_whose_limits_are_already_configured(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# Regression for #42657: token-limit discovery must not send GET /v1/models
|
||||
# to a deployment that already declares both max_input_tokens and
|
||||
# max_output_tokens. On scale-to-zero backends every such call cold-starts a
|
||||
# worker and bills real money to confirm values the operator already set.
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]})
|
||||
|
||||
router: Final = Router(model_list=[
|
||||
{
|
||||
"model_name": "fully-configured",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": "https://configured.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {
|
||||
"id": "fully-configured",
|
||||
"max_input_tokens": 2048,
|
||||
"max_output_tokens": 1024,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "needs-discovery",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": "https://discover.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "needs-discovery"},
|
||||
},
|
||||
])
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
responder: Final = Mock(side_effect=respond)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
|
||||
# Only the unconfigured deployment triggers a discovery request; the
|
||||
# fully-configured one is skipped entirely.
|
||||
discovery_hosts = {call.args[0].url.host for call in responder.call_args_list}
|
||||
assert discovery_hosts == {"discover.test"}
|
||||
assert responder.call_count == 1
|
||||
|
||||
# The configured deployment keeps its explicit limits; the discovered one
|
||||
# picks up the upstream card.
|
||||
assert router.get_configured_token_limits("fully-configured") == (2048, 1024)
|
||||
assert router.get_configured_token_limits("needs-discovery") == (4096, 4096)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_deployment_token_limits_configured_predicate() -> None:
|
||||
# Direct coverage for the guard predicate #42657 adds: discovery is skipped
|
||||
# only when BOTH per-request limits are explicitly set, so a partially
|
||||
# configured deployment still discovers the missing one.
|
||||
assert Router._deployment_token_limits_configured(
|
||||
{"max_input_tokens": 2048, "max_output_tokens": 1024}
|
||||
)
|
||||
assert not Router._deployment_token_limits_configured({"max_input_tokens": 2048})
|
||||
assert not Router._deployment_token_limits_configured({"max_output_tokens": 1024})
|
||||
assert not Router._deployment_token_limits_configured({})
|
||||
assert not Router._deployment_token_limits_configured(None)
|
||||
|
||||
|
||||
def test_should_not_pollute_shared_key_with_zero_cost_pricing():
|
||||
"""
|
||||
When deployment A has input_cost_per_token=0 and deployment B has no
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue