mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
fix(router): drop a tier param the routed target cannot take (#38622)
* fix(router): drop a tier param the routed target cannot take A complexity tier's litellm_params are an operator override applied to every request that tier routes, and they were written into the request kwargs unconditionally. When the tier set a param the target does not declare, get_optional_params raised UnsupportedParamsError before the request left the proxy, so the whole tier answered 400. The bundled Lite preset sets reasoning_effort on its complex tier, and four of the thirteen kimi-k3 map entries reject that param, so a router built from a first-party template failed on every complex prompt Filter the tier params at both store sites against what the group's deployments declare. The candidate set is asked of the module that raises rather than derived from a second list, so credentials, endpoint and transport controls are never at risk: base_url, timeout, default_headers, organization and deployment_id are not chat completion params and never reach that comparison. A param survives if any deployment could take it, since routing has not picked one yet, and it survives an unresolvable provider or an empty group, since a best-effort filter must not narrow what the request already did The skip list _check_valid_arg applies before rejecting a param now has one owner both it and the router read, so the two cannot drift * fix(router): honor allowed_openai_params when gating tier params * test(router): cover _declared_param_allowlist malformed declarations * fix(router): never ask an authenticating provider whether it takes a tier param Resolving github_copilot or chatgpt runs their OAuth device flow, so the capability question _deployment_accepts_param asks would freeze the event loop for minutes inside async_get_available_deployment. Promote register_model's local skip set to constants.PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO and fail open on those providers before any lookup * fix(utils): adopt a declared authenticating prefix instead of resolving it The tier-param guard alone was not enough: the savings baseline and the model-info funnels also resolve deployments during routing, and each resolution of github_copilot or chatgpt runs their OAuth device flow. declared_authenticating_provider gives every metadata funnel (get_supported_openai_params, _get_potential_model_names, _supports_factory, canonical_model) the resolver's answer by string, so the whole routing path answers without authenticating. A through-test drives async_get_available_deployment with a copilot deployment and records that no copilot resolution happens
This commit is contained in:
parent
72f1b3e969
commit
1e4d358f3f
10 changed files with 547 additions and 24 deletions
|
|
@ -629,6 +629,15 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"amazon_nova",
|
||||
]
|
||||
|
||||
# Resolving these providers runs an OAuth device flow (their provider info IS the login), so any
|
||||
# metadata or capability lookup against them can block for minutes waiting on a human.
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
|
||||
{
|
||||
"github_copilot",
|
||||
"chatgpt",
|
||||
}
|
||||
)
|
||||
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
|
||||
"openai",
|
||||
"azure",
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from typing import Final, cast
|
|||
from urllib.parse import urlparse
|
||||
|
||||
import litellm
|
||||
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
||||
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
match_routing_generalization,
|
||||
)
|
||||
|
|
@ -127,6 +127,18 @@ def handle_anthropic_text_model_custom_llm_provider(
|
|||
return model, custom_llm_provider
|
||||
|
||||
|
||||
def declared_authenticating_provider(model: str, custom_llm_provider: str | None = None) -> str | None:
|
||||
"""The authenticating provider this pair already names, or None.
|
||||
|
||||
get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, because their
|
||||
provider info includes the key it unlocks. For a metadata question that flow is pure hazard,
|
||||
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
|
||||
adopt the declaration instead of resolving.
|
||||
"""
|
||||
declared: Final = custom_llm_provider or model.split("/", 1)[0]
|
||||
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
|
||||
|
||||
|
||||
def get_llm_provider(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from typing import Final, Literal
|
|||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.types.utils import LlmProviders, LlmProvidersSet
|
||||
|
||||
|
||||
|
|
@ -30,6 +31,10 @@ def get_supported_openai_params(
|
|||
- List if custom_llm_provider is mapped
|
||||
- None if unmapped
|
||||
"""
|
||||
if not custom_llm_provider:
|
||||
custom_llm_provider = declared_authenticating_provider(
|
||||
model
|
||||
) # rebind-ok: resolving would run the provider's OAuth flow
|
||||
if not custom_llm_provider:
|
||||
try:
|
||||
custom_llm_provider = litellm.get_llm_provider(model=model)[1]
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
PTU_COST_ATTRIBUTION_ENV_VAR,
|
||||
|
|
@ -229,6 +230,7 @@ from litellm.types.utils import (
|
|||
StandardLoggingPayload,
|
||||
StandardLoggingRoutingDecision,
|
||||
Usage,
|
||||
all_litellm_params,
|
||||
shared_backend_model_info,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo as ModelMapInfo
|
||||
|
|
@ -243,6 +245,7 @@ from litellm.utils import (
|
|||
get_secret,
|
||||
get_utc_datetime,
|
||||
is_region_allowed,
|
||||
provider_rejectable_params,
|
||||
set_live_deployment_replay,
|
||||
)
|
||||
|
||||
|
|
@ -10833,6 +10836,114 @@ class Router:
|
|||
}
|
||||
return {**deployment, "model_info": model_info} # mutable-ok: DeploymentTypedDict rows are plain dicts
|
||||
|
||||
TIER_PARAMS_NEVER_DROPPED: Final = frozenset(all_litellm_params) | frozenset(
|
||||
{
|
||||
"additional_drop_params",
|
||||
"drop_params",
|
||||
"messages",
|
||||
"model",
|
||||
"extra_headers",
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
|
||||
declared: Final = params.get("allowed_openai_params")
|
||||
if not isinstance(declared, (list, tuple, set, frozenset)):
|
||||
return frozenset()
|
||||
return frozenset(entry for entry in declared if isinstance(entry, str))
|
||||
|
||||
@staticmethod
|
||||
def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param: str) -> bool:
|
||||
deployment_params: Final = deployment.get("litellm_params")
|
||||
if not deployment_params:
|
||||
return True
|
||||
if param in Router._declared_param_allowlist(deployment_params):
|
||||
return True
|
||||
if declared_authenticating_provider(
|
||||
str(deployment_params.get("model") or ""), deployment_params.get("custom_llm_provider")
|
||||
):
|
||||
return True
|
||||
deployment_model_info: Final = deployment.get("model_info")
|
||||
base_model: Final = (
|
||||
deployment_model_info.get("base_model") if deployment_model_info else None
|
||||
) or deployment_params.get("base_model")
|
||||
try:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=deployment_params.get("model") or group,
|
||||
custom_llm_provider=deployment_params.get("custom_llm_provider"),
|
||||
)
|
||||
supported: Final = litellm.get_supported_openai_params(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e
|
||||
)
|
||||
return True
|
||||
return supported is None or param in supported
|
||||
|
||||
def _tier_params_the_target_accepts(
|
||||
self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
"""Drop an OpenAI param that no deployment behind ``model`` declares.
|
||||
|
||||
A tier's litellm_params are an operator override applied to every request the tier routes,
|
||||
so one the target cannot take turns that whole tier into a 400 raised before the request
|
||||
leaves the proxy. The candidates are exactly what get_optional_params can reject, asked of
|
||||
the module that raises, so credentials and endpoint controls are never at risk.
|
||||
|
||||
TIER_PARAMS_NEVER_DROPPED is excluded on top of that, for two reasons. No provider lists a
|
||||
litellm control among its supported params, so "no deployment declares it" means litellm
|
||||
consumes it rather than that the target refuses it, and dropping one changes litellm's own
|
||||
behavior: dropping drop_params or additional_drop_params silently disables the sanitization
|
||||
the operator configured. Providers do list extra_headers, but it carries auth, tenancy and
|
||||
routing information, so sending fewer headers than configured is worse than today's error.
|
||||
Token ceilings stay for the same reason: a tier's max_tokens or max_completion_tokens is a
|
||||
cost bound, and dropping it would let a caller's own larger value through where today the
|
||||
mismatch fails loudly.
|
||||
|
||||
The trade this filter makes is a param for a working request, which is right for one that
|
||||
only shapes how the model answers and wrong for anything else.
|
||||
|
||||
A param survives if ANY deployment could take it, because routing has not chosen one yet,
|
||||
and it survives both an unresolvable provider and a group with no deployments, because a
|
||||
best-effort filter must never narrow what the request already did.
|
||||
|
||||
A github_copilot or chatgpt deployment counts as accepting everything, decided before any
|
||||
lookup: resolving either provider runs its OAuth device flow, so a capability question
|
||||
asked from the routing path can freeze the event loop for minutes waiting on a human.
|
||||
|
||||
allowed_openai_params is the documented escape hatch for an outdated or incomplete
|
||||
supported-params list: request-time validation extends the supported list with it before
|
||||
comparing. The filter asks the same question, so a param named by the allowlist on the tier
|
||||
overlay, the request, or a deployment's own litellm_params is never a drop candidate.
|
||||
"""
|
||||
deployments: Final = self.get_model_list(model_name=model) or ()
|
||||
if not deployments:
|
||||
return tier_params
|
||||
allowlisted: Final = self._declared_param_allowlist(tier_params) | self._declared_param_allowlist(
|
||||
request_kwargs
|
||||
)
|
||||
candidates: Final = provider_rejectable_params(tier_params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted
|
||||
unsupported: Final = frozenset(
|
||||
param
|
||||
for param in candidates
|
||||
if not any(self._deployment_accepts_param(deployment, model, param) for deployment in deployments)
|
||||
)
|
||||
if not unsupported:
|
||||
return tier_params
|
||||
verbose_router_logger.warning(
|
||||
"litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them",
|
||||
", ".join(sorted(unsupported)),
|
||||
model,
|
||||
)
|
||||
return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported})
|
||||
|
||||
def get_model_list(
|
||||
self, model_name: str | None = None, team_id: str | None = None
|
||||
) -> list[DeploymentTypedDict] | None:
|
||||
|
|
@ -11888,7 +11999,11 @@ class Router:
|
|||
model = pre_routing_hook_response.model
|
||||
messages = pre_routing_hook_response.messages
|
||||
if pre_routing_hook_response.litellm_params:
|
||||
request_kwargs.update(pre_routing_hook_response.litellm_params)
|
||||
request_kwargs.update(
|
||||
self._tier_params_the_target_accepts(
|
||||
model, pre_routing_hook_response.litellm_params, request_kwargs
|
||||
)
|
||||
)
|
||||
#########################################################
|
||||
|
||||
# Resolve the strategy and logger AFTER the pre-routing hook, since
|
||||
|
|
@ -11999,7 +12114,11 @@ class Router:
|
|||
model = pre_routing_hook_response.model
|
||||
messages = pre_routing_hook_response.messages
|
||||
if pre_routing_hook_response.litellm_params:
|
||||
request_kwargs.update(pre_routing_hook_response.litellm_params)
|
||||
request_kwargs.update(
|
||||
self._tier_params_the_target_accepts(
|
||||
model, pre_routing_hook_response.litellm_params, request_kwargs
|
||||
)
|
||||
)
|
||||
|
||||
# 2. Get healthy deployments
|
||||
healthy_deployments: Final = await self.async_get_healthy_deployments(
|
||||
|
|
|
|||
|
|
@ -54,9 +54,17 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
|
|||
A deployment may name its vendor in the model prefix or in a separate
|
||||
``custom_llm_provider``, and the bare name alone is not enough to price: it can
|
||||
resolve to a different vendor's rates, or to nothing at all.
|
||||
|
||||
A github_copilot or chatgpt candidate is qualified by string alone: resolving either
|
||||
provider runs its OAuth device flow, and for a declared pair the resolver's answer is
|
||||
the declaration itself, so asking it buys nothing but the block.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared is not None:
|
||||
return f"{declared}/{model.removeprefix(f'{declared}/')}"
|
||||
try:
|
||||
resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.constants import (
|
|||
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
OPENAI_EMBEDDING_PARAMS,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
|
|
@ -2555,10 +2556,19 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) ->
|
|||
Raises:
|
||||
Exception: If the given model is not found or there's an error in retrieval.
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
try:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared is not None:
|
||||
model = model.removeprefix(
|
||||
f"{declared}/"
|
||||
) # rebind-ok: mirrors get_llm_provider's split without its OAuth flow
|
||||
custom_llm_provider = declared # rebind-ok: same
|
||||
else:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
|
|
@ -2991,12 +3001,7 @@ def register_model(
|
|||
for _registered_key, _registered_value in _registrations.items():
|
||||
_runtime_registered_model_cost[_registered_key] = dict(_registered_value) # mutable-ok: caller-owned
|
||||
|
||||
# Providers that trigger side effects (e.g., OAuth flows) when get_model_info is called
|
||||
# Skip get_model_info for these providers during model registration
|
||||
_skip_get_model_info_providers: Final = {
|
||||
LlmProviders.GITHUB_COPILOT.value,
|
||||
LlmProviders.CHATGPT.value,
|
||||
}
|
||||
_skip_get_model_info_providers: Final = PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
|
||||
|
||||
for key, value in loaded_model_cost.items():
|
||||
## get model info ##
|
||||
|
|
@ -4150,17 +4155,11 @@ def get_optional_params(
|
|||
unsupported_params: Final = {}
|
||||
for k in non_default_params:
|
||||
if k not in supported_params:
|
||||
if k == "user" or k == "stream_options" or k == "stream":
|
||||
if k in PROVIDER_UNVALIDATED_PARAMS:
|
||||
continue
|
||||
if k == "n" and n == 1: # langchain sends n=1 as a default value
|
||||
continue # skip this param
|
||||
if (
|
||||
k == "max_retries"
|
||||
): # TODO: This is a patch. We support max retries for OpenAI, Azure. For non OpenAI LLMs we need to add support for max retries
|
||||
continue # skip this param
|
||||
# Always keeps this in elif code blocks
|
||||
else:
|
||||
unsupported_params[k] = non_default_params[k]
|
||||
unsupported_params[k] = non_default_params[k]
|
||||
|
||||
if unsupported_params:
|
||||
if litellm.drop_params is True or (drop_params is not None and drop_params is True):
|
||||
|
|
@ -4729,6 +4728,22 @@ def _apply_openai_param_overrides(optional_params: dict, non_default_params: dic
|
|||
return optional_params
|
||||
|
||||
|
||||
PROVIDER_UNVALIDATED_PARAMS: Final = frozenset({"user", "stream_options", "stream", "max_retries"})
|
||||
|
||||
|
||||
def provider_rejectable_params(passed_params: Mapping[str, object]) -> frozenset[str]:
|
||||
"""The params a provider can actually be rejected for, i.e. the ones _check_valid_arg compares
|
||||
against its supported list.
|
||||
|
||||
Anything outside this set never reaches that comparison. Endpoint and transport controls such as
|
||||
base_url, timeout, default_headers, organization and deployment_id are not chat completion
|
||||
params at all, so a caller filtering on "is this an OpenAI param" would discard configuration the
|
||||
request needs while never touching what the provider would have rejected.
|
||||
"""
|
||||
params: Final = dict(passed_params) # mutable-ok: get_non_default_params takes a dict
|
||||
return frozenset(get_non_default_params(params)) - PROVIDER_UNVALIDATED_PARAMS
|
||||
|
||||
|
||||
def get_non_default_params(passed_params: dict) -> dict:
|
||||
# filter out those parameters that were passed with non-default values
|
||||
non_default_params: Final = {
|
||||
|
|
@ -5544,6 +5559,8 @@ def _get_model_info_helper(
|
|||
"""
|
||||
Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
try:
|
||||
azure_llms: Final = {**litellm.azure_llms, **litellm.azure_embedding_models}
|
||||
if model in azure_llms:
|
||||
|
|
@ -5558,7 +5575,9 @@ def _get_model_info_helper(
|
|||
):
|
||||
model = model + "@latest"
|
||||
##########################
|
||||
potential_model_names: Final = _get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider)
|
||||
potential_model_names: Final = _get_potential_model_names(
|
||||
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
|
||||
)
|
||||
|
||||
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
|
||||
|
||||
|
|
|
|||
|
|
@ -173,3 +173,41 @@ def test_bedrock_converse_alias_keeps_nova_web_search_options():
|
|||
|
||||
assert nova_params is not None
|
||||
assert "web_search_options" in nova_params
|
||||
|
||||
|
||||
class TestDeclaredAuthenticatingProvider:
|
||||
"""github_copilot and chatgpt run an OAuth device flow inside get_llm_provider, so every
|
||||
metadata funnel must adopt a declared prefix instead of resolving it. A raising sentinel
|
||||
cannot prove the lookup was skipped, because these callers swallow resolver errors."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, provider, expected",
|
||||
[
|
||||
("github_copilot/gpt-4o", None, "github_copilot"),
|
||||
("chatgpt/gpt-5", None, "chatgpt"),
|
||||
("gpt-4o", "github_copilot", "github_copilot"),
|
||||
("openai/gpt-4o", None, None),
|
||||
("gpt-4o", "openai", None),
|
||||
],
|
||||
)
|
||||
def test_names_only_the_providers_whose_resolution_authenticates(self, model, provider, expected):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
assert declared_authenticating_provider(model, provider) == expected
|
||||
|
||||
@pytest.mark.parametrize("model", ["github_copilot/gpt-4o", "chatgpt/gpt-5"])
|
||||
def test_supported_params_never_resolve_an_authenticating_prefix(self, model, monkeypatch):
|
||||
import litellm
|
||||
|
||||
lookups: list = []
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
lookups.append((args, kwargs))
|
||||
raise RuntimeError("provider resolution must not run for an authenticating provider")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _record)
|
||||
|
||||
params = get_supported_openai_params(model=model)
|
||||
|
||||
assert params is not None
|
||||
assert lookups == []
|
||||
|
|
|
|||
|
|
@ -2213,14 +2213,14 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": {
|
||||
"model_name": "gpt-4o-mini",
|
||||
"model_name": "gpt-5-mini",
|
||||
"litellm_params": {"reasoning_effort": "xhigh"},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
|
||||
{"model_name": "gpt-5-mini", "litellm_params": {"model": "openai/gpt-5-mini"}},
|
||||
]
|
||||
)
|
||||
request_kwargs: Dict = {"reasoning_effort": "low"}
|
||||
|
|
@ -2231,9 +2231,66 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert deployment["model_name"] == "gpt-4o-mini"
|
||||
assert deployment["model_name"] == "gpt-5-mini"
|
||||
assert request_kwargs["reasoning_effort"] == "xhigh"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_never_resolves_an_authenticating_provider(self, monkeypatch, tmp_path):
|
||||
"""Resolving github_copilot runs its OAuth device flow, so the whole routing path must
|
||||
answer without it: the tier-param filter fails open, the savings baseline qualifies by
|
||||
string, and model info adopts the declared prefix. The recording wrapper raises for a
|
||||
copilot-directed resolution rather than calling through, so a regression fails on the
|
||||
recorded call instead of hanging the suite in a device-code poll."""
|
||||
import json
|
||||
import time
|
||||
|
||||
monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path))
|
||||
(tmp_path / "api-key.json").write_text(
|
||||
json.dumps({"token": "tid=test", "expires_at": int(time.time()) + 3600})
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "smart-router",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": {
|
||||
"model_name": "cop-mixed",
|
||||
"litellm_params": {"reasoning_effort": "high"},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
{"model_name": "cop-mixed", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}},
|
||||
{"model_name": "cop-mixed", "litellm_params": {"model": "github_copilot/gpt-4o"}},
|
||||
]
|
||||
)
|
||||
real_get_llm_provider = litellm.get_llm_provider
|
||||
copilot_resolutions: List = []
|
||||
|
||||
def _guarded(*args, **kwargs):
|
||||
target = str(kwargs.get("model") or (args[0] if args else "")) + str(kwargs.get("custom_llm_provider") or "")
|
||||
if "github_copilot" in target:
|
||||
copilot_resolutions.append(target)
|
||||
raise RuntimeError("routing must not resolve an authenticating provider")
|
||||
return real_get_llm_provider(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _guarded)
|
||||
request_kwargs: Dict = {}
|
||||
|
||||
deployment = await router.async_get_available_deployment(
|
||||
model="smart-router",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert deployment["model_name"] == "cop-mixed"
|
||||
assert request_kwargs["reasoning_effort"] == "high"
|
||||
assert copilot_resolutions == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_custom_pricing_is_not_applied_to_request_kwargs(self):
|
||||
"""Custom pricing on the alias prices the alias, not the tier deployment
|
||||
|
|
|
|||
|
|
@ -35,6 +35,31 @@ class TestCanonicalModel:
|
|||
def test_returns_none_for_a_name_no_provider_claims(self):
|
||||
assert canonical_model("") is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, provider, expected",
|
||||
[
|
||||
("github_copilot/gpt-4o", None, "github_copilot/gpt-4o"),
|
||||
("chatgpt/gpt-5", None, "chatgpt/gpt-5"),
|
||||
("gpt-4o", "github_copilot", "github_copilot/gpt-4o"),
|
||||
],
|
||||
)
|
||||
def test_never_resolves_a_provider_whose_lookup_authenticates(self, model, provider, expected, monkeypatch):
|
||||
"""Resolving github_copilot or chatgpt runs their OAuth device flow, so the baseline must
|
||||
qualify these by string alone. A raising sentinel cannot prove the lookup was skipped,
|
||||
because canonical_model swallows resolver errors into None."""
|
||||
import litellm
|
||||
|
||||
lookups: list = []
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
lookups.append((args, kwargs))
|
||||
raise RuntimeError("provider resolution must not run for an authenticating provider")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _record)
|
||||
|
||||
assert canonical_model(model, provider) == expected
|
||||
assert lookups == []
|
||||
|
||||
|
||||
class TestModelsForGroup:
|
||||
def test_resolves_a_group_to_the_models_its_deployments_call(self, parent):
|
||||
|
|
|
|||
|
|
@ -11213,3 +11213,234 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses(
|
|||
result is not "the call fails", so what to do about it stays each caller's policy.
|
||||
"""
|
||||
assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected)
|
||||
|
||||
|
||||
class TestTierParamsTheTargetAccepts:
|
||||
"""A tier's litellm_params are applied to every request that tier routes, so one the target
|
||||
cannot take raised UnsupportedParamsError before the request left the proxy, turning the whole
|
||||
tier into a 400."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def force_local_model_cost(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
|
||||
@staticmethod
|
||||
def _router(model: str) -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[{"model_name": "tiered", "litellm_params": {"model": model, "api_key": "sk-x"}}]
|
||||
)
|
||||
|
||||
def test_drops_a_param_no_deployment_declares(self):
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {}
|
||||
|
||||
def test_keeps_a_param_the_deployment_declares(self):
|
||||
router = self._router("fireworks_ai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"control, value",
|
||||
[
|
||||
("api_base", "https://example.invalid"),
|
||||
("api_key", "sk-tier"),
|
||||
("base_url", "https://example.invalid"),
|
||||
("timeout", 30),
|
||||
("default_headers", {"x-tier": "1"}),
|
||||
("organization", "org-tier"),
|
||||
("deployment_id", "dep-tier"),
|
||||
],
|
||||
)
|
||||
def test_keeps_credentials_and_transport_controls(self, control, value):
|
||||
"""These are not chat completion params, so get_optional_params never compares them against
|
||||
a provider's supported list. Filtering on "is this an OpenAI param" would discard the
|
||||
configuration the request needs while never touching what the provider would reject."""
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {control: value}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"control, value",
|
||||
[
|
||||
("additional_drop_params", ["seed"]),
|
||||
("drop_params", True),
|
||||
("allowed_openai_params", ["seed"]),
|
||||
("api_version", "2024-02-01"),
|
||||
("metadata", {"tier": "complex"}),
|
||||
],
|
||||
)
|
||||
def test_keeps_litellm_controls_the_provider_never_lists(self, control, value):
|
||||
"""No provider lists a litellm control among its supported params, so "no deployment
|
||||
declares it" means litellm consumes it, not that the target refuses it. Dropping
|
||||
drop_params or additional_drop_params would silently disable the operator's sanitization."""
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {control: value}
|
||||
|
||||
def test_tier_allowlist_protects_the_param_it_names(self):
|
||||
"""allowed_openai_params is the documented escape hatch for an incomplete supported-params
|
||||
list, and request-time validation extends the supported list with it, so a param the tier
|
||||
both sets and allowlists would never 400 and must not be dropped."""
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
"tiered", {"reasoning_effort": "max", "allowed_openai_params": ["reasoning_effort"]}, {}
|
||||
)
|
||||
|
||||
assert accepted == {"reasoning_effort": "max", "allowed_openai_params": ["reasoning_effort"]}
|
||||
|
||||
def test_request_allowlist_protects_the_param_it_names(self):
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
"tiered", {"reasoning_effort": "max"}, {"allowed_openai_params": ["reasoning_effort"]}
|
||||
)
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
def test_allowlist_protects_only_the_params_it_names(self):
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
"tiered", {"reasoning_effort": "max", "allowed_openai_params": ["seed"]}, {}
|
||||
)
|
||||
|
||||
assert accepted == {"allowed_openai_params": ["seed"]}
|
||||
|
||||
def test_declared_param_allowlist_ignores_malformed_declarations(self):
|
||||
"""A str is iterable, so without the type guard a YAML scalar mistake like
|
||||
allowed_openai_params: reasoning_effort would allowlist single characters."""
|
||||
assert litellm.Router._declared_param_allowlist({"allowed_openai_params": ["reasoning_effort", 3]}) == frozenset(
|
||||
{"reasoning_effort"}
|
||||
)
|
||||
assert litellm.Router._declared_param_allowlist({"allowed_openai_params": "reasoning_effort"}) == frozenset()
|
||||
assert litellm.Router._declared_param_allowlist({}) == frozenset()
|
||||
|
||||
def test_deployment_accepts_param_honors_deployment_allowlist(self):
|
||||
deployment = {
|
||||
"model_name": "x",
|
||||
"litellm_params": {"model": "novita/moonshotai/kimi-k3", "allowed_openai_params": ["reasoning_effort"]},
|
||||
}
|
||||
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True
|
||||
|
||||
def test_keeps_a_token_ceiling_the_provider_spells_differently(self):
|
||||
"""petals lists max_tokens but not max_completion_tokens. A tier ceiling in the unsupported
|
||||
spelling is a cost bound: dropping it would let a caller's larger max_tokens through where
|
||||
today the mismatch fails loudly."""
|
||||
router = self._router("petals/petals-team/StableBeluga2")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
"tiered", {"max_completion_tokens": 100, "reasoning_effort": "max"}, {}
|
||||
)
|
||||
|
||||
assert accepted == {"max_completion_tokens": 100}
|
||||
|
||||
def test_keeps_extra_headers_even_when_the_provider_omits_it(self):
|
||||
"""Several providers leave extra_headers out of their supported params, so the filter would
|
||||
drop it. Headers carry auth and tenancy, so sending fewer than the operator configured is
|
||||
worse than the error they already get."""
|
||||
router = self._router("ai21/jamba-1.5-mini")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
"tiered", {"extra_headers": {"x-tenant": "acme"}, "reasoning_effort": "max"}, {}
|
||||
)
|
||||
|
||||
assert accepted == {"extra_headers": {"x-tenant": "acme"}}
|
||||
|
||||
def test_keeps_a_param_any_deployment_in_the_group_declares(self):
|
||||
"""Routing has not picked a deployment yet, so one capable member keeps the param alive."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "tiered", "litellm_params": {"model": "novita/moonshotai/kimi-k3", "api_key": "k"}},
|
||||
{"model_name": "tiered", "litellm_params": {"model": "fireworks_ai/kimi-k3", "api_key": "k"}},
|
||||
]
|
||||
)
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
def test_deployment_accepts_param_honors_base_model(self):
|
||||
"""An azure deployment named after the deployment rather than the model carries the real
|
||||
model in base_model, and request-time mapping resolves capability through it, so the filter
|
||||
has to ask the same question or it drops a param the deployment accepts."""
|
||||
by_model_info = {
|
||||
"model_name": "x",
|
||||
"litellm_params": {"model": "azure/my-gpt5-deploy"},
|
||||
"model_info": {"base_model": "azure/gpt-5"},
|
||||
}
|
||||
by_litellm_params = {
|
||||
"model_name": "x",
|
||||
"litellm_params": {"model": "azure/my-gpt5-deploy", "base_model": "azure/gpt-5"},
|
||||
}
|
||||
without_hint = {"model_name": "x", "litellm_params": {"model": "azure/my-gpt5-deploy"}}
|
||||
|
||||
assert litellm.Router._deployment_accepts_param(by_model_info, "x", "reasoning_effort") is True
|
||||
assert litellm.Router._deployment_accepts_param(by_litellm_params, "x", "reasoning_effort") is True
|
||||
assert litellm.Router._deployment_accepts_param(without_hint, "x", "reasoning_effort") is False
|
||||
|
||||
def test_deployment_accepts_param_reads_the_provider(self):
|
||||
deployment = {"model_name": "x", "litellm_params": {"model": "fireworks_ai/kimi-k3"}}
|
||||
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True
|
||||
|
||||
def test_deployment_accepts_param_is_false_when_the_provider_omits_it(self):
|
||||
deployment = {"model_name": "x", "litellm_params": {"model": "novita/moonshotai/kimi-k3"}}
|
||||
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is False
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment",
|
||||
[{"model_name": "x"}, {"model_name": "x", "litellm_params": {}}, {"model_name": "x", "litellm_params": {"model": "not-a-real-provider/nope"}}],
|
||||
)
|
||||
def test_deployment_accepts_param_fails_open(self, deployment):
|
||||
"""An unresolvable deployment must not be the reason a param is dropped."""
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_params",
|
||||
[
|
||||
{"model": "github_copilot/gpt-4o"},
|
||||
{"model": "chatgpt/gpt-5"},
|
||||
{"model": "gpt-4o", "custom_llm_provider": "github_copilot"},
|
||||
],
|
||||
)
|
||||
def test_deployment_accepts_param_never_asks_a_provider_whose_lookup_authenticates(
|
||||
self, litellm_params, monkeypatch
|
||||
):
|
||||
"""Resolving github_copilot or chatgpt runs their OAuth device flow, so a capability
|
||||
question asked from the routing path can freeze the event loop for minutes waiting on a
|
||||
human. The deployment counts as accepting everything, and the lookup is never made: an
|
||||
exception-based sentinel cannot prove that, because the filter swallows exceptions into
|
||||
the same keep answer."""
|
||||
lookups: list = []
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
lookups.append((args, kwargs))
|
||||
raise RuntimeError("provider resolution must not run for an authenticating provider")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _record)
|
||||
deployment = {"model_name": "x", "litellm_params": litellm_params}
|
||||
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True
|
||||
assert lookups == []
|
||||
|
||||
def test_keeps_everything_for_an_unknown_group(self):
|
||||
"""An unresolvable target must never narrow what the request already did."""
|
||||
router = self._router("fireworks_ai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("no-such-group", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue