mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
fix(utils): adopt a declared authenticating prefix instead of resolving it
The tier-param guard alone was not enough: the savings baseline and the model-info funnels also resolve deployments during routing, and each resolution of github_copilot or chatgpt runs their OAuth device flow. declared_authenticating_provider gives every metadata funnel (get_supported_openai_params, _get_potential_model_names, _supports_factory, canonical_model) the resolver's answer by string, so the whole routing path answers without authenticating. A through-test drives async_get_available_deployment with a copilot deployment and records that no copilot resolution happens
This commit is contained in:
parent
b6e3fd0aa5
commit
7cc9b181ec
8 changed files with 167 additions and 10 deletions
|
|
@ -2,7 +2,7 @@ from typing import Final, cast
|
|||
from urllib.parse import urlparse
|
||||
|
||||
import litellm
|
||||
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
||||
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
match_routing_generalization,
|
||||
)
|
||||
|
|
@ -127,6 +127,18 @@ def handle_anthropic_text_model_custom_llm_provider(
|
|||
return model, custom_llm_provider
|
||||
|
||||
|
||||
def declared_authenticating_provider(model: str, custom_llm_provider: str | None = None) -> str | None:
|
||||
"""The authenticating provider this pair already names, or None.
|
||||
|
||||
get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, because their
|
||||
provider info includes the key it unlocks. For a metadata question that flow is pure hazard,
|
||||
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
|
||||
adopt the declaration instead of resolving.
|
||||
"""
|
||||
declared: Final = custom_llm_provider or model.split("/", 1)[0]
|
||||
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
|
||||
|
||||
|
||||
def get_llm_provider(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from typing import Final, Literal
|
|||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.types.utils import LlmProviders, LlmProvidersSet
|
||||
|
||||
|
||||
|
|
@ -30,6 +31,10 @@ def get_supported_openai_params(
|
|||
- List if custom_llm_provider is mapped
|
||||
- None if unmapped
|
||||
"""
|
||||
if not custom_llm_provider:
|
||||
custom_llm_provider = declared_authenticating_provider(
|
||||
model
|
||||
) # rebind-ok: resolving would run the provider's OAuth flow
|
||||
if not custom_llm_provider:
|
||||
try:
|
||||
custom_llm_provider = litellm.get_llm_provider(model=model)[1]
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ from litellm.constants import (
|
|||
DEFAULT_HEALTH_CHECK_INTERVAL,
|
||||
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
|
||||
DEFAULT_MAX_LRU_CACHE_SIZE,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -65,6 +64,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
PTU_COST_ATTRIBUTION_ENV_VAR,
|
||||
|
|
@ -10861,10 +10861,9 @@ class Router:
|
|||
return True
|
||||
if param in Router._declared_param_allowlist(deployment_params):
|
||||
return True
|
||||
declared_provider: Final = (
|
||||
deployment_params.get("custom_llm_provider") or str(deployment_params.get("model") or "").split("/", 1)[0]
|
||||
)
|
||||
if declared_provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO:
|
||||
if declared_authenticating_provider(
|
||||
str(deployment_params.get("model") or ""), deployment_params.get("custom_llm_provider")
|
||||
):
|
||||
return True
|
||||
deployment_model_info: Final = deployment.get("model_info")
|
||||
base_model: Final = (
|
||||
|
|
|
|||
|
|
@ -54,9 +54,17 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
|
|||
A deployment may name its vendor in the model prefix or in a separate
|
||||
``custom_llm_provider``, and the bare name alone is not enough to price: it can
|
||||
resolve to a different vendor's rates, or to nothing at all.
|
||||
|
||||
A github_copilot or chatgpt candidate is qualified by string alone: resolving either
|
||||
provider runs its OAuth device flow, and for a declared pair the resolver's answer is
|
||||
the declaration itself, so asking it buys nothing but the block.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared is not None:
|
||||
return f"{declared}/{model.removeprefix(f'{declared}/')}"
|
||||
try:
|
||||
resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline
|
||||
|
|
|
|||
|
|
@ -2556,10 +2556,19 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) ->
|
|||
Raises:
|
||||
Exception: If the given model is not found or there's an error in retrieval.
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
try:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared is not None:
|
||||
model = model.removeprefix(
|
||||
f"{declared}/"
|
||||
) # rebind-ok: mirrors get_llm_provider's split without its OAuth flow
|
||||
custom_llm_provider = declared # rebind-ok: same
|
||||
else:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
|
|
@ -5550,6 +5559,8 @@ def _get_model_info_helper(
|
|||
"""
|
||||
Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
try:
|
||||
azure_llms: Final = {**litellm.azure_llms, **litellm.azure_embedding_models}
|
||||
if model in azure_llms:
|
||||
|
|
@ -5564,7 +5575,9 @@ def _get_model_info_helper(
|
|||
):
|
||||
model = model + "@latest"
|
||||
##########################
|
||||
potential_model_names: Final = _get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider)
|
||||
potential_model_names: Final = _get_potential_model_names(
|
||||
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
|
||||
)
|
||||
|
||||
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
|
||||
|
||||
|
|
|
|||
|
|
@ -173,3 +173,41 @@ def test_bedrock_converse_alias_keeps_nova_web_search_options():
|
|||
|
||||
assert nova_params is not None
|
||||
assert "web_search_options" in nova_params
|
||||
|
||||
|
||||
class TestDeclaredAuthenticatingProvider:
|
||||
"""github_copilot and chatgpt run an OAuth device flow inside get_llm_provider, so every
|
||||
metadata funnel must adopt a declared prefix instead of resolving it. A raising sentinel
|
||||
cannot prove the lookup was skipped, because these callers swallow resolver errors."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, provider, expected",
|
||||
[
|
||||
("github_copilot/gpt-4o", None, "github_copilot"),
|
||||
("chatgpt/gpt-5", None, "chatgpt"),
|
||||
("gpt-4o", "github_copilot", "github_copilot"),
|
||||
("openai/gpt-4o", None, None),
|
||||
("gpt-4o", "openai", None),
|
||||
],
|
||||
)
|
||||
def test_names_only_the_providers_whose_resolution_authenticates(self, model, provider, expected):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
|
||||
assert declared_authenticating_provider(model, provider) == expected
|
||||
|
||||
@pytest.mark.parametrize("model", ["github_copilot/gpt-4o", "chatgpt/gpt-5"])
|
||||
def test_supported_params_never_resolve_an_authenticating_prefix(self, model, monkeypatch):
|
||||
import litellm
|
||||
|
||||
lookups: list = []
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
lookups.append((args, kwargs))
|
||||
raise RuntimeError("provider resolution must not run for an authenticating provider")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _record)
|
||||
|
||||
params = get_supported_openai_params(model=model)
|
||||
|
||||
assert params is not None
|
||||
assert lookups == []
|
||||
|
|
|
|||
|
|
@ -2234,6 +2234,63 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
assert deployment["model_name"] == "gpt-5-mini"
|
||||
assert request_kwargs["reasoning_effort"] == "xhigh"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_never_resolves_an_authenticating_provider(self, monkeypatch, tmp_path):
|
||||
"""Resolving github_copilot runs its OAuth device flow, so the whole routing path must
|
||||
answer without it: the tier-param filter fails open, the savings baseline qualifies by
|
||||
string, and model info adopts the declared prefix. The recording wrapper raises for a
|
||||
copilot-directed resolution rather than calling through, so a regression fails on the
|
||||
recorded call instead of hanging the suite in a device-code poll."""
|
||||
import json
|
||||
import time
|
||||
|
||||
monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path))
|
||||
(tmp_path / "api-key.json").write_text(
|
||||
json.dumps({"token": "tid=test", "expires_at": int(time.time()) + 3600})
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "smart-router",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": {
|
||||
"model_name": "cop-mixed",
|
||||
"litellm_params": {"reasoning_effort": "high"},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
{"model_name": "cop-mixed", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}},
|
||||
{"model_name": "cop-mixed", "litellm_params": {"model": "github_copilot/gpt-4o"}},
|
||||
]
|
||||
)
|
||||
real_get_llm_provider = litellm.get_llm_provider
|
||||
copilot_resolutions: List = []
|
||||
|
||||
def _guarded(*args, **kwargs):
|
||||
target = str(kwargs.get("model") or (args[0] if args else "")) + str(kwargs.get("custom_llm_provider") or "")
|
||||
if "github_copilot" in target:
|
||||
copilot_resolutions.append(target)
|
||||
raise RuntimeError("routing must not resolve an authenticating provider")
|
||||
return real_get_llm_provider(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _guarded)
|
||||
request_kwargs: Dict = {}
|
||||
|
||||
deployment = await router.async_get_available_deployment(
|
||||
model="smart-router",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert deployment["model_name"] == "cop-mixed"
|
||||
assert request_kwargs["reasoning_effort"] == "high"
|
||||
assert copilot_resolutions == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_custom_pricing_is_not_applied_to_request_kwargs(self):
|
||||
"""Custom pricing on the alias prices the alias, not the tier deployment
|
||||
|
|
|
|||
|
|
@ -35,6 +35,31 @@ class TestCanonicalModel:
|
|||
def test_returns_none_for_a_name_no_provider_claims(self):
|
||||
assert canonical_model("") is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, provider, expected",
|
||||
[
|
||||
("github_copilot/gpt-4o", None, "github_copilot/gpt-4o"),
|
||||
("chatgpt/gpt-5", None, "chatgpt/gpt-5"),
|
||||
("gpt-4o", "github_copilot", "github_copilot/gpt-4o"),
|
||||
],
|
||||
)
|
||||
def test_never_resolves_a_provider_whose_lookup_authenticates(self, model, provider, expected, monkeypatch):
|
||||
"""Resolving github_copilot or chatgpt runs their OAuth device flow, so the baseline must
|
||||
qualify these by string alone. A raising sentinel cannot prove the lookup was skipped,
|
||||
because canonical_model swallows resolver errors into None."""
|
||||
import litellm
|
||||
|
||||
lookups: list = []
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
lookups.append((args, kwargs))
|
||||
raise RuntimeError("provider resolution must not run for an authenticating provider")
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _record)
|
||||
|
||||
assert canonical_model(model, provider) == expected
|
||||
assert lookups == []
|
||||
|
||||
|
||||
class TestModelsForGroup:
|
||||
def test_resolves_a_group_to_the_models_its_deployments_call(self, parent):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue