mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
fix(proxy): never run OAuth device flows when resolving model names
Resolving github_copilot/chatgpt names through get_llm_provider runs the provider's OAuth device flow synchronously on the event loop. Adopt the declared provider in PatternMatchRouter.get_pattern, which the auth layer's zero-cost budget check walks on every request against wildcard routers, and in /utils/supported_openai_params.
This commit is contained in:
parent
11a7471902
commit
779b3010d4
4 changed files with 127 additions and 16 deletions
|
|
@ -12478,11 +12478,21 @@ async def supported_openai_params(model: str):
|
|||
--header 'Authorization: Bearer sk-1234'
|
||||
```
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
global llm_router
|
||||
try:
|
||||
resolved_models: Final = llm_router.resolved_litellm_models(model) if llm_router is not None else ()
|
||||
litellm_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=resolved_models[0] if resolved_models else model
|
||||
resolved_models: Final = (
|
||||
llm_router.resolved_litellm_models(model)
|
||||
if llm_router is not None and declared_authenticating_provider(model) is None
|
||||
else ()
|
||||
)
|
||||
target_model: Final = resolved_models[0] if resolved_models else model
|
||||
declared_provider: Final = declared_authenticating_provider(target_model)
|
||||
litellm_model, custom_llm_provider = (
|
||||
(target_model.removeprefix(f"{declared_provider}/"), declared_provider)
|
||||
if declared_provider is not None
|
||||
else litellm.get_llm_provider(model=target_model)[:2]
|
||||
)
|
||||
return {
|
||||
"supported_openai_params": litellm.get_supported_openai_params(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from re import Match
|
|||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider
|
||||
|
||||
|
||||
class PatternUtils:
|
||||
|
|
@ -215,18 +215,17 @@ class PatternMatchRouter:
|
|||
Returns:
|
||||
bool: True if pattern exists, False otherwise
|
||||
"""
|
||||
if custom_llm_provider is None:
|
||||
try:
|
||||
(
|
||||
_,
|
||||
custom_llm_provider,
|
||||
_,
|
||||
_,
|
||||
) = get_llm_provider(model=model)
|
||||
except Exception:
|
||||
# get_llm_provider raises exception when provider is unknown
|
||||
pass
|
||||
return self.route(model) or self.route(f"{custom_llm_provider}/{model}")
|
||||
provider: Final = (
|
||||
custom_llm_provider or declared_authenticating_provider(model) or self._resolved_provider(model)
|
||||
)
|
||||
return self.route(model) or self.route(f"{provider}/{model}")
|
||||
|
||||
@staticmethod
|
||||
def _resolved_provider(model: str) -> str | None:
|
||||
try:
|
||||
return get_llm_provider(model=model)[1]
|
||||
except Exception: # noqa: BLE001 # get_llm_provider raises when the provider is unknown; the name then routes as-is
|
||||
return None
|
||||
|
||||
def get_deployments_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> list[dict]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -9,11 +9,13 @@ Pins (PR2):
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.router_utils import pattern_match_deployments
|
||||
|
||||
from .conftest import normalize # type: ignore[import-not-found]
|
||||
|
||||
|
|
@ -145,6 +147,55 @@ def test_supported_openai_params_resolves_router_alias(client, auth_as, monkeypa
|
|||
assert "max_tokens" in response.json()["supported_openai_params"]
|
||||
|
||||
|
||||
def test_supported_openai_params_never_runs_oauth_for_authenticating_providers(client, auth_as, monkeypatch, tmp_path):
|
||||
"""Regression: github_copilot/chatgpt names answer from their declaration; resolving them
|
||||
through ``get_llm_provider`` would run the provider's OAuth device flow and block the event loop."""
|
||||
monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path))
|
||||
(tmp_path / "access-token").write_text("fake-access-token")
|
||||
(tmp_path / "api-key.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"token": "fake-api-key",
|
||||
"expires_at": 4102444800,
|
||||
"endpoints": {"api": "https://api.githubcopilot.com"},
|
||||
}
|
||||
)
|
||||
)
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "copilot-alias",
|
||||
"litellm_params": {"model": "github_copilot/gpt-4o"},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*"},
|
||||
},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
||||
resolution_attempts: list[str] = []
|
||||
|
||||
def _oauth_tripwire(model, *args, **kwargs):
|
||||
resolution_attempts.append(model)
|
||||
raise AssertionError("get_llm_provider would run the OAuth device flow")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire)
|
||||
monkeypatch.setattr(pattern_match_deployments, "get_llm_provider", _oauth_tripwire)
|
||||
expected = litellm.get_supported_openai_params(model="gpt-4o", custom_llm_provider="github_copilot")
|
||||
|
||||
with auth_as():
|
||||
via_alias = client.get("/utils/supported_openai_params", params={"model": "copilot-alias"})
|
||||
via_direct_name = client.get("/utils/supported_openai_params", params={"model": "github_copilot/gpt-4o"})
|
||||
|
||||
assert via_alias.status_code == 200
|
||||
assert via_alias.json() == {"supported_openai_params": expected}
|
||||
assert via_direct_name.status_code == 200
|
||||
assert via_direct_name.json() == {"supported_openai_params": expected}
|
||||
assert resolution_attempts == []
|
||||
|
||||
|
||||
def test_supported_openai_params_invalid_model(client, auth_as, monkeypatch):
|
||||
"""Pins ``GET /utils/supported_openai_params`` (error: unknown model)."""
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
"""Behavior pins for ``litellm/router_utils/pattern_match_deployments.py``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from litellm.router_utils import pattern_match_deployments
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
|
||||
def _wildcard_deployment(model_name: str) -> dict:
|
||||
return {"model_name": model_name, "litellm_params": {"model": model_name}}
|
||||
|
||||
|
||||
def _matched_models(matches: list[dict] | None) -> list[str]:
|
||||
return [deployment["litellm_params"]["model"] for deployment in matches or []]
|
||||
|
||||
|
||||
def test_get_pattern_never_resolves_declared_authenticating_providers(monkeypatch):
|
||||
"""Regression: resolving a github_copilot/chatgpt name through ``get_llm_provider`` runs the
|
||||
provider's OAuth device flow; the auth layer walks every wildcard router on every request, so
|
||||
a single metadata lookup for an unserved name would block the proxy's event loop."""
|
||||
resolution_attempts: list[str] = []
|
||||
|
||||
def _oauth_tripwire(model, *args, **kwargs):
|
||||
resolution_attempts.append(model)
|
||||
raise AssertionError("get_llm_provider would run the OAuth device flow")
|
||||
|
||||
monkeypatch.setattr(pattern_match_deployments, "get_llm_provider", _oauth_tripwire)
|
||||
|
||||
unmatched_router = PatternMatchRouter()
|
||||
unmatched_router.add_pattern("anthropic/*", _wildcard_deployment("anthropic/*"))
|
||||
assert unmatched_router.get_pattern("github_copilot/gpt-4o") is None
|
||||
|
||||
matched_router = PatternMatchRouter()
|
||||
matched_router.add_pattern("github_copilot/*", _wildcard_deployment("github_copilot/*"))
|
||||
assert _matched_models(matched_router.get_pattern("github_copilot/gpt-4o")) == ["github_copilot/gpt-4o"]
|
||||
assert _matched_models(matched_router.get_pattern("gpt-4o", custom_llm_provider="github_copilot")) == [
|
||||
"github_copilot/gpt-4o"
|
||||
]
|
||||
|
||||
assert resolution_attempts == []
|
||||
|
||||
|
||||
def test_get_pattern_still_resolves_unqualified_names(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
pattern_match_deployments,
|
||||
"get_llm_provider",
|
||||
lambda model, **kwargs: (model, "openai", None, None),
|
||||
)
|
||||
router = PatternMatchRouter()
|
||||
router.add_pattern("openai/*", _wildcard_deployment("openai/*"))
|
||||
assert _matched_models(router.get_pattern("gpt-4o")) == ["openai/gpt-4o"]
|
||||
Loading…
Add table
Reference in a new issue