scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-04 07:27:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 299

Merge branch 'litellm_openrouter_live_model_discovery' of https://github.com/norkelbyte/litellm into litellm_openrouter_catalog_limits

Browse source
This commit is contained in:
mateo-berri 2026-10-30 01:31:07 -07:00
commit 69ff47d335
15 changed files with 1040 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
litellm/llms/openrouter/common_utils.py
View file

@ -1,5 +1,92 @@
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
from pydantic import BaseModel, ConfigDict
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
_DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1"
_CHAT_CATALOG_ROUTE: Final = "models/user"
_CHAT_COMPLETIONS_PATH: Final = "/chat/completions"
_BATCH_ONLY_SUFFIX: Final = ":batch"
_CATALOG_TIMEOUT: Final = httpx.Timeout(10.0)
class OpenRouterException(BaseLLMException):
pass
class _CatalogModel(BaseModel):
model_config = ConfigDict(frozen=True)
id: str
class _Catalog(BaseModel):
model_config = ConfigDict(frozen=True)
data: tuple[_CatalogModel, ...]
def _chat_catalog_url(api_base: str) -> str:
return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE)
class OpenRouterModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return (
api_key
or litellm.api_key
or litellm.openrouter_key
or get_secret_str("OPENROUTER_API_KEY")
or get_secret_str("OR_API_KEY")
)
@staticmethod
def get_api_base(api_base: str | None = None) -> str:
return api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or _DEFAULT_API_BASE
@staticmethod
def get_base_model(model: str) -> str:
return model.removeprefix("openrouter/")
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: BaseLLMModelInfo.validate_environment returns a dict
resolved_key: Final = self.get_api_key(api_key)
if resolved_key is None:
return {**headers}
return {**headers, "Authorization": f"Bearer {resolved_key}"}
def get_models(
self, api_key: str | None = None, api_base: str | None = None
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
resolved_key: Final = self.get_api_key(api_key)
if resolved_key is None:
raise ValueError("Set OPENROUTER_API_KEY or the deployment's api_key to list OpenRouter models")
headers: Final = self.validate_environment(
headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key
)
response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters
url=_chat_catalog_url(self.get_api_base(api_base)), headers=headers, timeout=_CATALOG_TIMEOUT
)
response.raise_for_status()
return [
f"openrouter/{model.id}"
for model in _Catalog.model_validate_json(response.content).data
if not model.id.endswith(_BATCH_ONLY_SUFFIX)
]

90
litellm/proxy/auth/model_checks.py
View file

@ -1,12 +1,15 @@
# What is this?
## Common checks for /v1/models and `/model/info`
import asyncio
import contextvars
import copy
import functools
import itertools
import re
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from types import MappingProxyType
from typing import Final
from typing import Final, TypeVar
import litellm
from litellm._logging import verbose_proxy_logger
@ -30,6 +33,17 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
_MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing")
_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value})
_Listed: Final = TypeVar("_Listed")
async def run_model_listing(listing: Callable[[], _Listed]) -> _Listed:
context: Final = contextvars.copy_context()
return await asyncio.get_running_loop().run_in_executor(_MODEL_LISTING_POOL, context.run, listing)
def _check_wildcard_routing(model: str) -> bool:
"""
@ -209,6 +223,7 @@ def get_complete_model_list(
include_model_access_groups: bool | None = False,
only_model_access_groups: bool | None = False,
team_id: str | None = None,
expand_wildcard_routes: bool = True,
) -> list[str]:
"""Logic for returning complete model list for a given key + team pair"""
@ -254,11 +269,10 @@ def get_complete_model_list(
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
team_id=team_id,
expand_wildcard_routes=expand_wildcard_routes,
)
complete_model_list: Final = unique_models + all_wildcard_models
return complete_model_list
return list(dict.fromkeys(unique_models + all_wildcard_models))
def _hydrate_litellm_credential_name(
@ -425,6 +439,15 @@ def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str |
return functools.cache(wildcard_routed_models)
def _deployed_by_name(llm_router: Router | None) -> Callable[[str, str | None], bool]:
if llm_router is None:
return lambda _model, _team_id: False
registry: Final = llm_router.registry()
deployed_names: Final = frozenset(registry.names(kind="group_set"))
team_deployed_names: Final = frozenset(registry.index("team_model"))
return lambda model, team_id: model in deployed_names or (team_id, model) in team_deployed_names
def _routes_to_model(
routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None
) -> Callable[[str, str], bool]:
@ -455,7 +478,11 @@ def get_known_models_from_wildcard(
wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1)
except ValueError: # safely fail
return []
deployment_pattern_naming_listed_ids: Final = _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
deployment_pattern_naming_listed_ids: Final = (
wildcard_model_to_expand
if wildcard_model_to_expand != wildcard_model
else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
)
# Use provider from litellm_params when available, otherwise from wildcard prefix
# (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router)
@ -514,6 +541,39 @@ def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_pro
return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}"
@functools.lru_cache(maxsize=256)
def _wildcard_name_regex(name_pattern: str) -> re.Pattern[str]:
return re.compile(PatternMatchRouter().pattern_to_regex(name_pattern))
def _routed_deployment_model(name_pattern: str, listed_name: str, deployment_model: str) -> str:
name_match: Final = _wildcard_name_regex(name_pattern).match(listed_name)
if name_match is None:
return listed_name
return PatternMatchRouter.set_deployment_model_name(
matched_pattern=name_match, litellm_deployment_litellm_model=deployment_model
)
def _names_listed_under_wildcard(
deployment: Mapping[str, object],
wildcard_pattern: str,
litellm_params: LiteLLM_Params | None,
routed_models: Callable[[str, str | None], frozenset[str]] | None,
deployed_by_name: Callable[[str, str | None], bool],
) -> tuple[str, ...]:
team_id: Final = _deployment_team_id(deployment)
return tuple(
name
for name in get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
litellm_params=litellm_params,
routes_to_model=_routes_to_model(routed_models, team_id),
)
if not deployed_by_name(name, team_id)
)
def expand_wildcard_deployments_for_model_info(
deployments: list[dict[str, object]],
llm_router: Router | None = None,
@ -527,6 +587,7 @@ def expand_wildcard_deployments_for_model_info(
"""
expanded: Final[list[dict[str, object]]] = []
routed_models: Final = _cached_wildcard_routing(llm_router)
deployed_by_name: Final = _deployed_by_name(llm_router)
for deployment in deployments:
model_name = str(deployment.get("model_name") or "")
raw_params = deployment.get("litellm_params")
@ -551,10 +612,8 @@ def expand_wildcard_deployments_for_model_info(
except Exception:
expanded.append(deployment)
continue
expanded_names = get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
litellm_params=litellm_params,
routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)),
expanded_names = _names_listed_under_wildcard(
deployment, wildcard_pattern, litellm_params, routed_models, deployed_by_name
)
if not expanded_names:
expanded.append(deployment)
@ -565,17 +624,24 @@ def expand_wildcard_deployments_for_model_info(
row["model_name"] = name
params = row.get("litellm_params")
if isinstance(params, dict):
params["model"] = name
params["model"] = _routed_deployment_model(model_name, name, litellm_model)
expanded.append(row)
return expanded
def _known_models_without_a_deployment(wildcard_model: str) -> Sequence[str]:
if wildcard_model.partition("/")[0] in _PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT:
return ()
return get_known_models_from_wildcard(wildcard_model=wildcard_model, litellm_params=None)
def _get_wildcard_models(
unique_models: list[str],
return_wildcard_routes: bool | None = False,
llm_router: Router | None = None,
team_id: str | None = None,
expand_wildcard_routes: bool = True,
) -> list[str]:
models_to_remove: Final = set()
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
@ -586,6 +652,8 @@ def _get_wildcard_models(
all_wildcard_models.append(model)
models_to_remove.add(model)
if not expand_wildcard_routes:
continue
model_list = llm_router.registry().deployments(model, team_id=team_id) if llm_router else None
if model_list:
@ -598,7 +666,7 @@ def _get_wildcard_models(
)
)
else:
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
all_wildcard_models.extend(_known_models_without_a_deployment(model))
for model in models_to_remove:
unique_models.remove(model)

53
litellm/proxy/openai_files_endpoints/common_utils.py
View file

@ -1,4 +1,5 @@
import base64
import itertools
import logging
import mimetypes
import re
@ -541,7 +542,8 @@ def get_team_provider_credentials(
listings on the team's own provider account/key instead of a shared
global one.
2. Fallback: any deployment the caller is granted access to for this
provider, expanding wildcard routes and the all-proxy-models sentinel.
provider, matching wildcard grants against deployed model names without
listing any provider's catalog, and expanding the all-proxy-models sentinel.
Skipped when ``include_accessible_deployments`` is False.
Credential lookup is scoped to both the team's allowlist and the key's own
@ -559,6 +561,7 @@ def get_team_provider_credentials(
return None
from litellm.proxy._types import SpecialModelNames
from litellm.proxy.auth.auth_checks import is_model_allowed_by_pattern
from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models
team_id: Final = user_api_key_dict.team_id
@ -645,28 +648,36 @@ def get_team_provider_credentials(
grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models
effective_team_models: Final = [] if grants_all_models else team_models
models_to_try: Final = list(
dict.fromkeys(
get_complete_model_list(
key_models=list(key_model_allowlist),
team_models=effective_team_models,
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=True,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=True,
team_id=team_id,
)
granted_models: Final = get_complete_model_list(
key_models=list(key_model_allowlist),
team_models=effective_team_models,
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=True,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=True,
team_id=team_id,
expand_wildcard_routes=False,
)
deployments_granted_by_wildcard: Final = (
model_name
for model_name in proxy_model_list
if any(
is_model_allowed_by_pattern(model=model_name, allowed_model_pattern=granted) for granted in granted_models
)
)
for model_name in models_to_try:
credentials = _provider_credentials(model_name)
if credentials is not None:
return credentials
return None
return next(
(
credentials
for credentials in map(
_provider_credentials, dict.fromkeys(itertools.chain(granted_models, deployments_granted_by_wildcard))
)
if credentials is not None
),
None,
)
HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*")

59
litellm/proxy/proxy_server.py
View file

@ -199,6 +199,7 @@ from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import (
PRICING_OVERRIDES_KEY,
CountContextManagement,
LlmProviders,
ModelResponse,
ModelResponseStream,
TextCompletionResponse,
@ -443,6 +444,7 @@ from litellm.proxy.auth.model_checks import (
get_key_models,
get_mcp_server_ids,
get_team_models,
run_model_listing,
)
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports
@ -12455,20 +12457,23 @@ async def model_list(
# Get complete model list including wildcard routes if requested
from litellm.proxy.auth.model_checks import get_complete_model_list
all_models = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[
*proxy_model_list,
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
],
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=include_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups or False,
only_model_access_groups=only_model_access_groups or False,
all_models = await run_model_listing(
partial(
get_complete_model_list,
key_models=[],
team_models=[],
proxy_model_list=[
*proxy_model_list,
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
],
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=include_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups or False,
only_model_access_groups=only_model_access_groups or False,
)
)
expanded_undiscoverable_names: Final = undiscoverable_model_names(
@ -14715,6 +14720,8 @@ from litellm.responses.litellm_completion_transformation.count_request import (
with_provider_file_ids,
)
_PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO: Final = frozenset({LlmProviders.OPENROUTER})
def _with_client_oauth_token(
counter: BaseTokenCounter | None, client_oauth_token: str | None
@ -14750,14 +14757,13 @@ def _get_provider_token_counter(
)
# Switch case pattern using existing get_provider_model_info
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
# Convert string provider to LlmProviders enum
llm_provider_enum: Final = LlmProviders(custom_llm_provider)
# Add more provider mappings as needed
if llm_provider_enum:
if llm_provider_enum not in _PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO:
provider_model_info: Final = ProviderConfigManager.get_provider_model_info(
model=full_model, provider=llm_provider_enum
)
@ -17909,7 +17915,7 @@ async def model_info_v1(
alias_models: Final = copy.deepcopy(registry.deployments(source="alias"))
all_models.extend(alias_models)
all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router)
all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router))
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
user_api_key_dict=user_api_key_dict,
@ -18247,14 +18253,17 @@ async def model_group_info(
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
)
all_models_str: Final = (
get_complete_model_list(
key_models=(),
team_models=(),
proxy_model_list=llm_router.registry().names(),
user_model=user_model,
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
return_wildcard_routes=False,
llm_router=llm_router,
await run_model_listing(
partial(
get_complete_model_list,
key_models=(),
team_models=(),
proxy_model_list=llm_router.registry().names(),
user_model=user_model,
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
return_wildcard_routes=False,
llm_router=llm_router,
)
)
if is_proxy_admin
else await get_available_models_for_user(

55
litellm/proxy/utils.py
View file

@ -9506,7 +9506,7 @@ async def _agent_access_group_visible_models(
) -> frozenset[str] | None:
"""Models an agent key may still list once its attached access groups cap it, ``None`` when
nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on."""
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_listing
if not user_api_key_dict.agent_id:
return None
@ -9527,17 +9527,20 @@ async def _agent_access_group_visible_models(
if not granted:
return frozenset()
return frozenset(
get_complete_model_list(
key_models=granted,
team_models=(),
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
team_id=team_id,
await run_model_listing(
partial(
get_complete_model_list,
key_models=granted,
team_models=(),
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
team_id=team_id,
)
)
)
@ -9582,6 +9585,7 @@ async def get_available_models_for_user(
get_complete_model_list,
get_key_models,
get_team_models,
run_model_listing,
)
# Get proxy model list and access groups
@ -9662,18 +9666,21 @@ async def get_available_models_for_user(
)
# Get complete model list
complete_models: Final = get_complete_model_list(
key_models=granted_key_models,
team_models=granted_team_models,
proxy_model_list=proxy_model_list,
user_model=user_model,
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
only_model_access_groups=only_model_access_groups,
team_id=effective_team_id,
complete_models: Final = await run_model_listing(
partial(
get_complete_model_list,
key_models=granted_key_models,
team_models=granted_team_models,
proxy_model_list=proxy_model_list,
user_model=user_model,
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
only_model_access_groups=only_model_access_groups,
team_id=effective_team_id,
)
)
all_models: Final = (
list(

4
litellm/router_utils/pattern_match_deployments.py
View file

@ -279,7 +279,9 @@ class PatternMatchRouter:
PatternMatchRouter.set_deployment_model_name(
matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"]
)
for deployment in self.patterns[pattern]
for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts
pattern, ()
)
)

81
litellm/utils.py
View file

@ -8723,11 +8723,13 @@ class AvailableModelsCache(InMemoryCache):
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
result: Final = cast(list[str] | None, self.get_cache(cache_key))
result: Final = self._cached(cache_key)
if result is not None:
return copy.deepcopy(result)
return result
if isinstance(result, Sequence):
return list(result)
if self._cached(_failed_listing_key(cache_key)) is not None:
return []
return None
def set_cached_model_info(
self,
@ -8737,12 +8739,46 @@ class AvailableModelsCache(InMemoryCache):
):
"""Set cached model info"""
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
self.set_cache(cache_key, copy.deepcopy(available_models))
self._store(cache_key, copy.deepcopy(available_models))
def set_failed_model_listing(
self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None, ttl_seconds: float
) -> None:
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
self._store(_failed_listing_key(cache_key), True, ttl=ttl_seconds)
def listing_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock:
return _provider_listing_lock(self._get_cache_key(custom_llm_provider, litellm_params))
def _cached(self, key: str) -> Sequence[str] | bool | None:
return cast(Sequence[str] | bool | None, self.get_cache(key))
def _store(self, key: str, value: object, ttl: float | None = None) -> None:
self.set_cache(key, value, ttl=ttl)
def _failed_listing_key(cache_key: str) -> str:
return f"{cache_key}:failed"
_PROVIDER_LISTING_LOCK_GUARD: Final = threading.Lock()
@lru_cache(maxsize=1024)
def _cached_provider_listing_lock(cache_key: str) -> threading.RLock:
return threading.RLock()
def _provider_listing_lock(cache_key: str) -> threading.RLock:
with _PROVIDER_LISTING_LOCK_GUARD:
return _cached_provider_listing_lock(cache_key)
# Global cache instance
_model_cache: Final = AvailableModelsCache()
_FAILED_PROVIDER_LISTING_TTL_SECONDS: Final = 30
def _infer_valid_provider_from_env_vars(
custom_llm_provider: str | None = None,
@ -8790,20 +8826,27 @@ def _get_valid_models_from_provider_api(
custom_llm_provider: str,
litellm_params: LiteLLM_Params | None = None,
) -> list[str]:
try:
cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
with _model_cache.listing_lock(custom_llm_provider, litellm_params):
try:
cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
if cached_result is not None:
return cached_result
models: Final = provider_config.discover_models(
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
)
if cached_result is not None:
return cached_result
models: Final = provider_config.discover_models(
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
)
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
return []
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
return models
except httpx.TimeoutException as e:
verbose_logger.warning("Error getting valid models: %s", e)
_model_cache.set_failed_model_listing(
custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS
)
return []
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
return []
def get_valid_models(
@ -10464,6 +10507,10 @@ class ProviderConfigManager:
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
return AzureFoundryModelInfo(model=model)
elif LlmProviders.OPENROUTER == provider:
from litellm.llms.openrouter.common_utils import OpenRouterModelInfo
return OpenRouterModelInfo()
return None
@staticmethod

159
tests/unit/llms/openrouter/test_openrouter_common_utils.py Normal file
View file

@ -0,0 +1,159 @@
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.llms.openrouter.common_utils import OpenRouterModelInfo
DEFAULT_BASE: Final = "https://openrouter.ai/api/v1"
KEY_SOURCES: Final = ("api_key", "litellm.api_key", "litellm.openrouter_key", "OPENROUTER_API_KEY", "OR_API_KEY")
BASE_SOURCES: Final = ("api_base", "litellm.api_base", "OPENROUTER_API_BASE")
@pytest.fixture(autouse=True)
def _no_ambient_openrouter_settings(monkeypatch: pytest.MonkeyPatch):
for env_var in ("OPENROUTER_API_BASE", "OPENROUTER_API_KEY", "OR_API_KEY"):
monkeypatch.delenv(env_var, raising=False)
monkeypatch.setattr(litellm, "api_key", None)
monkeypatch.setattr(litellm, "api_base", None)
monkeypatch.setattr(litellm, "openrouter_key", None)
def _catalog(*model_ids: str) -> httpx.Response:
return httpx.Response(200, json={"data": [{"id": model_id, "name": model_id} for model_id in model_ids]})
def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None:
if source.startswith("litellm."):
monkeypatch.setattr(litellm, source.removeprefix("litellm."), value)
else:
monkeypatch.setenv(source, value)
def test_get_models_lists_only_the_chat_catalog_under_openrouter_keeping_the_org_namespace():
with respx.mock() as respx_mock:
respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5")
)
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
"openrouter/openai/gpt-5.4-mini",
"openrouter/anthropic/claude-haiku-5-5",
]
assert [call.request.url.path for call in respx_mock.calls] == ["/api/v1/models/user"]
@respx.mock
def test_get_models_drops_batch_only_ids_and_keeps_every_other_variant():
respx.get(f"{DEFAULT_BASE}/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini:batch", "openai/gpt-5.4-mini", "qwen/qwen3-coder:free")
)
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
"openrouter/openai/gpt-5.4-mini",
"openrouter/qwen/qwen3-coder:free",
]
@pytest.mark.parametrize(
"api_base",
[
"https://gateway.example/openrouter/api/v1/",
"https://gateway.example/openrouter/api/v1/chat/completions",
"https://gateway.example/openrouter/api/v1/chat/completions/",
],
)
@respx.mock
def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(api_base: str):
chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini")
)
assert OpenRouterModelInfo().get_models(api_key="sk-or-deployment", api_base=api_base) == [
"openrouter/openai/gpt-5.4-mini"
]
assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-deployment"
@pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES)
@respx.mock
def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order(
monkeypatch: pytest.MonkeyPatch, winner: int
):
for source in KEY_SOURCES[winner + 1 :]:
_configure(monkeypatch, source, f"sk-or-from-{source}")
if winner > 0:
_configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner")
chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder"))
assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [
"openrouter/qwen/qwen3-coder"
]
assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-winner"
@pytest.mark.parametrize("winner", range(len(BASE_SOURCES) + 1), ids=(*BASE_SOURCES, "default"))
@respx.mock
def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order(
monkeypatch: pytest.MonkeyPatch, winner: int
):
for source in BASE_SOURCES[winner + 1 :]:
_configure(monkeypatch, source, f"https://{source.lower().replace('.', '-')}.example/api/v1")
if 0 < winner < len(BASE_SOURCES):
_configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1")
expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE
respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder"))
assert OpenRouterModelInfo().get_models(
api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None
) == ["openrouter/qwen/qwen3-coder"]
def test_get_models_without_any_key_raises_before_calling_openrouter():
with respx.mock(assert_all_called=False) as respx_mock:
chat_route: Final = respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(
return_value=_catalog("openai/gpt-5.4-mini")
)
with pytest.raises(ValueError, match="OPENROUTER_API_KEY"):
OpenRouterModelInfo().get_models()
assert not chat_route.called
@respx.mock
def test_get_models_raises_when_the_catalog_answers_an_error():
respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=httpx.Response(401, json={"error": {"code": 401}}))
with pytest.raises(httpx.HTTPStatusError):
OpenRouterModelInfo().get_models(api_key="sk-or-revoked")
def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-env")
headers: Final = {"X-Title": "my-app"}
assert OpenRouterModelInfo().validate_environment(
headers=headers, model="openrouter/openai/gpt-5.4-mini", messages=(), optional_params={}, litellm_params={}
) == {"X-Title": "my-app", "Authorization": "Bearer sk-or-env"}
assert headers == {"X-Title": "my-app"}
def test_validate_environment_without_any_key_sends_no_authorization():
assert OpenRouterModelInfo().validate_environment(
headers={"X-Title": "my-app"},
model="openrouter/openai/gpt-5.4-mini",
messages=(),
optional_params={},
litellm_params={},
) == {"X-Title": "my-app"}
@pytest.mark.parametrize(
("model", "base_model"),
[("openrouter/openai/gpt-5.4-mini", "openai/gpt-5.4-mini"), ("openai/gpt-5.4-mini", "openai/gpt-5.4-mini")],
)
def test_get_base_model_drops_only_the_openrouter_prefix(model: str, base_model: str):
assert OpenRouterModelInfo.get_base_model(model) == base_model

191
tests/unit/proxy/auth/test_model_checks.py
View file

@ -7,6 +7,8 @@ import httpx
import pytest
import respx
from litellm import Router
def test_get_team_models_for_all_models_and_team_only_models():
from litellm.proxy.auth.model_checks import get_team_models
@ -1261,6 +1263,195 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it
assert get_known_models_from_wildcard("bedrock/anthropic.*", deployment) == []
@pytest.mark.parametrize(
("wildcard", "deployment_model", "listed_ids"),
[
(
"openrouter/*",
"openrouter/*",
["openrouter/openai/gpt-5.4-mini", "openrouter/anthropic/claude-haiku-5-5"],
),
("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]),
("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]),
("*", "openrouter/*", ["openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5"]),
],
)
@respx.mock
def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace(
monkeypatch: pytest.MonkeyPatch, wildcard: str, deployment_model: str, listed_ids: list[str]
) -> None:
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
respx.get("https://openrouter.ai/api/v1/models/user").mock(
return_value=httpx.Response(
200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]}
)
)
assert (
get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}"))
== listed_ids
)
@respx.mock
def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_no_environment_catalog(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm
from litellm import Router
from litellm.proxy.auth.model_checks import get_complete_model_list
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-environment-only")
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(
return_value=httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
)
router: Final = Router(
model_list=[{"model_name": "or-team/*", "litellm_params": {"model": "openrouter/*", "api_key": "sk-or-team"}}]
)
listed: Final = get_complete_model_list(
key_models=["openrouter/*"],
team_models=[],
proxy_model_list=router.model_names,
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
assert (listed, catalog.call_count) == ([], 0)
@contextmanager
def _openai_catalog_behind_a_catch_all(
monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str]
) -> Generator[Router, None, None]:
import litellm
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
with respx.mock(assert_all_called=False) as respx_mock:
respx_mock.get("https://api.openai.com/v1/models").mock(
return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "gpt-4o-mini"}]})
)
yield Router(
model_list=[
{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-openai-catch-all"}},
{
"model_name": concrete_name,
"litellm_params": {"model": "openrouter/openai/gpt-4o", "api_key": "sk-or-gpt-4o"},
"model_info": dict(concrete_model_info),
},
]
)
@pytest.mark.parametrize(
("concrete_name", "concrete_model_info", "team_id"),
[
("gpt-4o", {}, None),
("model_name_team-x_gpt-4o", {"team_id": "team-x", "team_public_model_name": "gpt-4o"}, "team-x"),
],
)
def test_catch_all_wildcard_lists_a_name_a_concrete_deployment_serves_once(
monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str], team_id: str | None
) -> None:
from litellm.proxy.auth.model_checks import get_complete_model_list
with _openai_catalog_behind_a_catch_all(monkeypatch, concrete_name, concrete_model_info) as router:
listed: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=router.get_model_names(team_id=team_id),
user_model=None,
infer_model_from_keys=False,
llm_router=router,
team_id=team_id,
)
assert sorted(listed) == ["gpt-4o", "gpt-4o-mini"]
def test_catch_all_wildcard_model_info_keeps_only_the_concrete_row_for_a_deployed_name(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info
with _openai_catalog_behind_a_catch_all(monkeypatch, "gpt-4o", {}) as router:
rows: Final = expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router
)
assert sorted((row["model_name"], row["litellm_params"]["model"]) for row in rows) == [
("gpt-4o", "openrouter/openai/gpt-4o"),
("gpt-4o-mini", "openai/gpt-4o-mini"),
]
@contextmanager
def _openrouter_catalog_beside_a_named_deployment(
monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str
) -> Generator[Router, None, None]:
import litellm
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
with respx.mock(assert_all_called=False) as respx_mock:
respx_mock.get("https://openrouter.ai/api/v1/models/user").mock(
return_value=httpx.Response(
200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]}
)
)
yield Router(
model_list=[
{
"model_name": wildcard_name,
"litellm_params": {"model": "openrouter/*", "api_key": f"sk-or-wildcard-{wildcard_name}"},
},
{
"model_name": named,
"litellm_params": {"model": "openrouter/openai/gpt-5.4-mini", "api_key": "sk-or-named"},
},
]
)
@pytest.mark.parametrize(
("wildcard_name", "named"),
[("openrouter/*", "openrouter/openai/gpt-5.4-mini"), ("or-team/*", "or-team/openai/gpt-5.4-mini")],
)
def test_provider_wildcard_lists_a_name_a_named_deployment_serves_once(
monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str
) -> None:
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
with _openrouter_catalog_beside_a_named_deployment(monkeypatch, wildcard_name, named) as router:
listed: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=list(router.model_names),
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
rows: Final = expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router
)
vendor_prefix: Final = wildcard_name.removesuffix("*")
assert sorted(listed) == [f"{vendor_prefix}anthropic/claude-haiku-5-5", named]
assert sorted((row["model_name"], row["litellm_params"]["api_key"]) for row in rows) == [
(f"{vendor_prefix}anthropic/claude-haiku-5-5", f"sk-or-wildcard-{wildcard_name}"),
(named, "sk-or-named"),
]
@respx.mock
def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model(
monkeypatch: pytest.MonkeyPatch,

112
tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py
View file

@ -3,10 +3,13 @@ from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import respx
from fastapi import HTTPException
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
import litellm
from litellm import Router
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
@ -852,3 +855,112 @@ def test_a_team_keeps_its_first_config_file_openai_deployment_over_older_db_ones
assert credentials is not None
assert credentials["api_key"] == "deployment-z-key"
@pytest.mark.parametrize(
("custom_llm_provider", "openrouter_key", "api_key"),
[("openai", "sk-or-files-openai", "openai-key"), ("openrouter", "sk-or-files-openrouter", "sk-or-files-openrouter")],
)
def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_catalog(
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, openrouter_key: str, api_key: str
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
router: Final = Router(
model_list=[
{
"model_name": "openrouter/*",
"litellm_params": {
"model": "openrouter/*",
"api_key": openrouter_key,
"api_base": "https://openrouter.ai/api/v1",
},
},
{"model_name": "gpt-5.4-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "openai-key"}},
]
)
with respx.mock(assert_all_called=False) as respx_mock:
catalog_route: Final = respx_mock.get("https://openrouter.ai/api/v1/models/user").mock(
return_value=httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
)
credentials: Final = get_team_provider_credentials(
llm_router=router,
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user"),
custom_llm_provider=custom_llm_provider,
include_accessible_deployments=True,
)
assert credentials is not None
assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key)
assert catalog_route.call_count == 0
@pytest.mark.parametrize(
("key_models", "deployments", "custom_llm_provider", "api_key"),
[
(
["anthropic/*"],
(("anthropic/claude-*", "anthropic/claude-*", "anthropic-narrow-key"),),
"anthropic",
"anthropic-narrow-key",
),
(["anthropic/claude-3*"], (("anthropic/*", "anthropic/*", "anthropic-wide-key"),), "anthropic", "anthropic-wide-key"),
(["openrouter/*"], (("openrouter/openai/*", "openrouter/openai/*", "sk-or-narrow"),), "openrouter", "sk-or-narrow"),
(
["team/*"],
(("team/*", "openrouter/*", "sk-or-team"), ("team/openai/*", "openai/*", "openai-team-key")),
"openai",
"openai-team-key",
),
],
)
def test_a_key_wildcard_finds_the_deployment_it_grants_without_listing_any_provider_catalog(
monkeypatch: pytest.MonkeyPatch,
key_models: list[str],
deployments: tuple[tuple[str, str, str], ...],
custom_llm_provider: str,
api_key: str,
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
router: Final = Router(
model_list=[
{"model_name": model_name, "litellm_params": {"model": model, "api_key": deployment_key}}
for model_name, model, deployment_key in deployments
]
)
with respx.mock(assert_all_called=False) as respx_mock:
catalog_requests: Final = respx_mock.route().mock(return_value=httpx.Response(200, json={"data": []}))
credentials: Final = get_team_provider_credentials(
llm_router=router,
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=key_models),
custom_llm_provider=custom_llm_provider,
include_accessible_deployments=True,
)
assert credentials is not None
assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key)
assert catalog_requests.call_count == 0
@pytest.mark.parametrize(
("deployed_name", "api_key"), [("gpt-4o-mini", None), ("openai/gpt-4o-mini", "openai-deployment-key")]
)
def test_a_provider_wildcard_grant_finds_only_deployments_named_under_it(deployed_name: str, api_key: str | None):
router: Final = Router(
model_list=[
{
"model_name": deployed_name,
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "openai-deployment-key"},
}
]
)
credentials: Final = get_team_provider_credentials(
llm_router=router,
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["openai/*"]),
custom_llm_provider="openai",
include_accessible_deployments=True,
)
assert (None if credentials is None else credentials["api_key"]) == api_key

12
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -550,15 +550,12 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke
deployment = {
"model_name": "*",
"litellm_params": {"model": "openai/*"},
"litellm_params": {"model": "openai/*", "api_key": "sk-star"},
}
router = route_entry_points_to_stubs(MagicMock())
router.get_model_access_groups = MagicMock(return_value={})
router.get_model_names = MagicMock(return_value=["*"])
router.get_model_list = MagicMock(return_value=[deployment])
router = litellm.Router(model_list=[deployment])
monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models)
expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info([deployment])
expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info(router.get_model_list(), router)
allowed_model_names = await proxy_server._get_v1_model_info_allowed_model_names_with_access_groups(
user_api_key_dict=UserAPIKeyAuth(
api_key="sk-test",
@ -575,7 +572,8 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke
allowed_model_names=allowed_model_names,
)
assert [model["model_name"] for model in result] == ["openai/gpt-4o"]
assert [(model["model_name"], model["litellm_params"]["model"]) for model in result] == [("gpt-4o", "openai/gpt-4o")]
assert router.registry().names("gpt-4o", kind="wildcard") == frozenset({"openai/gpt-4o"})
# ---------------------------------------------------------------------------

73
tests/unit/proxy/proxy_server/test_routes_models.py
View file

@ -9,10 +9,16 @@ Pins (PR2):
from __future__ import annotations
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
import respx
import litellm
from litellm import Router
@ -541,6 +547,73 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1"
@dataclass(frozen=True, slots=True)
class _CatalogHeldUntilTheLoopRuns:
requested: threading.Event = field(default_factory=threading.Event)
released: threading.Event = field(default_factory=threading.Event)
def answer(self, _request: httpx.Request) -> httpx.Response:
self.requested.set()
if not self.released.wait(timeout=30):
return httpx.Response(504)
return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
async def release_from_the_loop(self) -> None:
assert await asyncio.to_thread(self.requested.wait, 30)
self.released.set()
@pytest.mark.parametrize("scope", [None, "expand"])
def test_v1_models_fetches_a_live_openrouter_catalog_without_holding_the_event_loop_or_its_default_thread_pool(
monkeypatch: pytest.MonkeyPatch, scope: str | None
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(
proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "openrouter/*",
"litellm_params": {
"model": "openrouter/*",
"api_key": f"sk-or-v1-models-off-loop-{scope}",
"api_base": OPENROUTER_BASE,
},
}
]
),
)
catalog: Final = _CatalogHeldUntilTheLoopRuns()
async def listed_ids_on_a_one_thread_default_pool() -> list[str]:
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
listing: Final = asyncio.ensure_future(
proxy_server.model_list(
user_api_key_dict=UserAPIKeyAuth(
api_key="sk-test-key", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
),
scope=scope,
)
)
await catalog.release_from_the_loop()
response: Final = await listing
return [str(model["id"]) for model in response["data"]]
with respx.mock() as respx_mock:
catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer)
listed_ids: Final = asyncio.run(listed_ids_on_a_one_thread_default_pool())
assert catalog_route.calls.last.response.status_code == 200
assert listed_ids == ["openrouter/openai/gpt-5.4-mini"]
def _priced_jev_models() -> frozenset[str]:
return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/"))

33
tests/unit/proxy/test_proxy_token_counter.py
View file

@ -179,6 +179,39 @@ async def test_token_counting_model_not_in_model_list():
assert response.model_used == "special-alias"
@pytest.mark.asyncio
async def test_asking_the_provider_endpoint_leaves_an_openrouter_deployments_count_unchanged(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
litellm.proxy.proxy_server,
"llm_router",
Router(
model_list=[
{
"model_name": "or-gpt",
"litellm_params": {
"model": "openai/gpt-4o",
"custom_llm_provider": "openrouter",
"api_key": "sk-or-token-count",
},
}
]
),
)
request: Final = TokenCountRequest(
model="or-gpt", messages=[{"role": "user", "content": "Count these tokens, ünïcödé 你好 🙂"}]
)
via_provider_endpoint: Final = await token_counter(request=request, call_endpoint=True)
counted_locally: Final = await token_counter(request=request, call_endpoint=False)
assert (via_provider_endpoint.total_tokens, via_provider_endpoint.model_used) == (
counted_locally.total_tokens,
counted_locally.model_used,
)
@pytest.mark.asyncio
async def test_gpt_token_counting():
"""

67
tests/unit/proxy/utils/helpers/test_model_access.py
View file

@ -1,10 +1,17 @@
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
import respx
from fastapi import HTTPException
import litellm
from litellm import ModelResponse
from litellm import ModelResponse, Router
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import (
create_model_info_response,
@ -650,3 +657,61 @@ async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups(
resolve_agent_ceiling=_agent_ceiling(None),
)
assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"])
OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1"
@dataclass(frozen=True, slots=True)
class _CatalogHeldUntilTheLoopRuns:
requested: threading.Event = field(default_factory=threading.Event)
released: threading.Event = field(default_factory=threading.Event)
def answer(self, _request: httpx.Request) -> httpx.Response:
self.requested.set()
if not self.released.wait(timeout=30):
return httpx.Response(504)
return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
async def release_from_the_loop(self) -> None:
assert await asyncio.to_thread(self.requested.wait, 30)
self.released.set()
def test_get_available_models_for_user_fetches_a_live_catalog_without_holding_the_event_loop_or_its_default_pool(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
catalog: Final = _CatalogHeldUntilTheLoopRuns()
router: Final = Router(
model_list=[
{
"model_name": "openrouter/*",
"litellm_params": {
"model": "openrouter/*",
"api_key": "sk-or-available-models-off-loop",
"api_base": OPENROUTER_BASE,
},
}
]
)
async def list_models_on_a_one_thread_default_pool() -> list[str]:
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
listing: Final = asyncio.ensure_future(
get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"),
llm_router=router,
general_settings={},
user_model=None,
)
)
await catalog.release_from_the_loop()
return await listing
with respx.mock() as respx_mock:
catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer)
models: Final = asyncio.run(list_models_on_a_one_thread_default_pool())
assert catalog_route.calls.last.response.status_code == 200
assert models == ["openrouter/openai/gpt-5.4-mini"]

71
tests/unit/test_utils.py
View file

@ -9269,6 +9269,77 @@ def reset_mock_cache() -> None:
_model_cache.flush_cache()
@respx.mock
def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_listing(reset_mock_cache: None) -> None:
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(
side_effect=httpx.ReadTimeout("catalog hung")
)
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage")
listings: Final = tuple(
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
for _ in range(2)
)
assert (listings, catalog.call_count) == (([], []), 1)
@respx.mock
def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None:
respx.get("https://openrouter.ai/api/v1/models/user").mock(
side_effect=(httpx.Response(503), httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}))
)
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-recovers")
listings: Final = tuple(
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
for _ in range(2)
)
assert listings == ([], ["openrouter/anthropic/claude-haiku-5-5"])
@respx.mock
def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it(
reset_mock_cache: None,
) -> None:
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings")
first_request: Final = iter((True, False))
def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response:
if next(first_request):
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
raise httpx.ReadTimeout("catalog hung", request=request)
return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing)
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
assert get_valid_models(
check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment
) == ["openrouter/anthropic/claude-haiku-5-5"]
@respx.mock
def test_get_valid_models_asks_a_provider_once_when_many_listings_start_together(reset_mock_cache: None) -> None:
def slow_catalog(request: httpx.Request) -> httpx.Response:
threading.Event().wait(0.2)
return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=slow_catalog)
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-many-listings")
def list_models(_: int) -> tuple[str, ...]:
return tuple(
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
)
with ThreadPoolExecutor(max_workers=8) as pool:
listings: Final = frozenset(pool.map(list_models, range(8)))
assert (listings, catalog.call_count) == (frozenset({("openrouter/anthropic/claude-haiku-5-5",)}), 1)
@pytest.fixture
def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(