mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
fix(router): never ask an authenticating provider whether it takes a tier param
Resolving github_copilot or chatgpt runs their OAuth device flow, so the capability question _deployment_accepts_param asks would freeze the event loop for minutes inside async_get_available_deployment. Promote register_model's local skip set to constants.PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO and fail open on those providers before any lookup
This commit is contained in:
parent
37e1bfecb1
commit
b6e3fd0aa5
4 changed files with 49 additions and 6 deletions
|
|
@ -629,6 +629,15 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"amazon_nova",
|
||||
]
|
||||
|
||||
# Resolving these providers runs an OAuth device flow (their provider info IS the login), so any
|
||||
# metadata or capability lookup against them can block for minutes waiting on a human.
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
|
||||
{
|
||||
"github_copilot",
|
||||
"chatgpt",
|
||||
}
|
||||
)
|
||||
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
|
||||
"openai",
|
||||
"azure",
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.constants import (
|
|||
DEFAULT_HEALTH_CHECK_INTERVAL,
|
||||
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
|
||||
DEFAULT_MAX_LRU_CACHE_SIZE,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -10860,6 +10861,11 @@ class Router:
|
|||
return True
|
||||
if param in Router._declared_param_allowlist(deployment_params):
|
||||
return True
|
||||
declared_provider: Final = (
|
||||
deployment_params.get("custom_llm_provider") or str(deployment_params.get("model") or "").split("/", 1)[0]
|
||||
)
|
||||
if declared_provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO:
|
||||
return True
|
||||
deployment_model_info: Final = deployment.get("model_info")
|
||||
base_model: Final = (
|
||||
deployment_model_info.get("base_model") if deployment_model_info else None
|
||||
|
|
@ -10908,6 +10914,10 @@ class Router:
|
|||
and it survives both an unresolvable provider and a group with no deployments, because a
|
||||
best-effort filter must never narrow what the request already did.
|
||||
|
||||
A github_copilot or chatgpt deployment counts as accepting everything, decided before any
|
||||
lookup: resolving either provider runs its OAuth device flow, so a capability question
|
||||
asked from the routing path can freeze the event loop for minutes waiting on a human.
|
||||
|
||||
allowed_openai_params is the documented escape hatch for an outdated or incomplete
|
||||
supported-params list: request-time validation extends the supported list with it before
|
||||
comparing. The filter asks the same question, so a param named by the allowlist on the tier
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.constants import (
|
|||
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
OPENAI_EMBEDDING_PARAMS,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
|
|
@ -2991,12 +2992,7 @@ def register_model(
|
|||
for _registered_key, _registered_value in _registrations.items():
|
||||
_runtime_registered_model_cost[_registered_key] = dict(_registered_value) # mutable-ok: caller-owned
|
||||
|
||||
# Providers that trigger side effects (e.g., OAuth flows) when get_model_info is called
|
||||
# Skip get_model_info for these providers during model registration
|
||||
_skip_get_model_info_providers: Final = {
|
||||
LlmProviders.GITHUB_COPILOT.value,
|
||||
LlmProviders.CHATGPT.value,
|
||||
}
|
||||
_skip_get_model_info_providers: Final = PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
|
||||
|
||||
for key, value in loaded_model_cost.items():
|
||||
## get model info ##
|
||||
|
|
|
|||
|
|
@ -11390,6 +11390,34 @@ class TestTierParamsTheTargetAccepts:
|
|||
"""An unresolvable deployment must not be the reason a param is dropped."""
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_params",
|
||||
[
|
||||
{"model": "github_copilot/gpt-4o"},
|
||||
{"model": "chatgpt/gpt-5"},
|
||||
{"model": "gpt-4o", "custom_llm_provider": "github_copilot"},
|
||||
],
|
||||
)
|
||||
def test_deployment_accepts_param_never_asks_a_provider_whose_lookup_authenticates(
|
||||
self, litellm_params, monkeypatch
|
||||
):
|
||||
"""Resolving github_copilot or chatgpt runs their OAuth device flow, so a capability
|
||||
question asked from the routing path can freeze the event loop for minutes waiting on a
|
||||
human. The deployment counts as accepting everything, and the lookup is never made: an
|
||||
exception-based sentinel cannot prove that, because the filter swallows exceptions into
|
||||
the same keep answer."""
|
||||
lookups: list = []
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
lookups.append((args, kwargs))
|
||||
raise RuntimeError("provider resolution must not run for an authenticating provider")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _record)
|
||||
deployment = {"model_name": "x", "litellm_params": litellm_params}
|
||||
|
||||
assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True
|
||||
assert lookups == []
|
||||
|
||||
def test_keeps_everything_for_an_unknown_group(self):
|
||||
"""An unresolvable target must never narrow what the request already did."""
|
||||
router = self._router("fireworks_ai/kimi-k3")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue