mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(openrouter)!: list OpenRouter's live catalog for openrouter/* when check_provider_endpoint is on (#48820)
* fix(openrouter): list OpenRouter's live catalog for openrouter/* when check_provider_endpoint is on * fix(openrouter): list models off the event loop and keep only chat-callable catalog ids The model-list builders are sync, and with check_provider_endpoint on they call OpenRouter's catalog with a 10 second timeout and no caching of failures. They ran on the event loop inside /v1/models, /model/info, /model_group/info and the per-user listing, so one slow catalog froze every request on the worker. Those calls now run in a worker thread with asyncio.to_thread File, batch and RAG credential lookups only need each name's deployment, and a wildcard that is itself a deployment sits in the list ahead of its expansions and resolves its own credentials. get_complete_model_list takes expand_deployed_wildcard_routes, which that lookup sets to False, so a deployed wildcard never fetches its provider catalog there. A key wildcard wider than any deployment, like anthropic/* over a deployed anthropic/claude-*, still expands so it finds the matching deployment's credentials. Every other caller keeps the default The pattern router's model lookup tolerates a pattern removed between its two reads of the pattern table, which a deployment delete on the event loop can now do while a listing runs in a thread Listing now reads only OpenRouter's models/user catalog. OpenRouter's own /models list carries no embedding models, the OpenAI-format /v1/models has no mode field so chat pickers cannot tell embeddings apart, /model_group/info labels them mode chat, and one failing catalog emptied the whole list. Ids ending in :batch are dropped because chat completions answers 404 for them, while :free and every other variant stay. An api_base that already ends in /chat/completions, which chat accepts as is, has that suffix stripped before the catalog route is joined * fix(proxy): keep model listing off the shared thread pool and out of credential lookups Listings run on their own thread pool so many slow catalogs cannot starve the default executor that chat and Responses calls share. File and batch credential lookups match deployed names against a key's wildcard grants instead of listing any provider catalog. The token counter keeps OpenRouter deployments on their previous path, and a catch-all "*" over provider/* lists bare ids that route back through it * test(proxy): give the off-loop listing tests a generous deadlock guard and typed ids * fix(proxy): keep routed models in wildcard model info, skip key-only OpenRouter catalogs, cache failed listings briefly * fix(utils): cache failed model listings under their own key so a late failure never hides a good listing * fix(proxy): remember only timed-out provider listings so a provider that failed fast is listed again * fix(proxy): ask each provider catalog once per worker, keep deployed names out of catch-all ids, and match file credentials by name Listings that start together for one deployment now wait for the first catalog request. A catch-all wildcard no longer lists a name a deployment serves by name. A key wildcard only finds file and batch credentials on deployments named under it, so a bare name keeps the default credentials as before. * fix(proxy): keep the listing failure log line the same for timed-out listings * refactor(proxy): fold the cached listing lookup back into the provider listing function * fix(proxy): list each model id once and keep a named deployment's row over a wildcard's * fix(proxy): drop a wildcard's model info row when named deployments serve every name it lists --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
norkelbyte
GitHub
parent
6ad4f9cfdf
commit
5898a42f5a
15 changed files with 1055 additions and 104 deletions
|
|
@ -1,5 +1,92 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
_DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1"
|
||||
_CHAT_CATALOG_ROUTE: Final = "models/user"
|
||||
_CHAT_COMPLETIONS_PATH: Final = "/chat/completions"
|
||||
_BATCH_ONLY_SUFFIX: Final = ":batch"
|
||||
_CATALOG_TIMEOUT: Final = httpx.Timeout(10.0)
|
||||
|
||||
|
||||
class OpenRouterException(BaseLLMException):
|
||||
pass
|
||||
|
||||
|
||||
class _CatalogModel(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
class _Catalog(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
data: tuple[_CatalogModel, ...]
|
||||
|
||||
|
||||
def _chat_catalog_url(api_base: str) -> str:
|
||||
return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE)
|
||||
|
||||
|
||||
class OpenRouterModelInfo(BaseLLMModelInfo):
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.openrouter_key
|
||||
or get_secret_str("OPENROUTER_API_KEY")
|
||||
or get_secret_str("OR_API_KEY")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
return api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or _DEFAULT_API_BASE
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> str:
|
||||
return model.removeprefix("openrouter/")
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: BaseLLMModelInfo.validate_environment returns a dict
|
||||
resolved_key: Final = self.get_api_key(api_key)
|
||||
if resolved_key is None:
|
||||
return {**headers}
|
||||
return {**headers, "Authorization": f"Bearer {resolved_key}"}
|
||||
|
||||
def get_models(
|
||||
self, api_key: str | None = None, api_base: str | None = None
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
resolved_key: Final = self.get_api_key(api_key)
|
||||
if resolved_key is None:
|
||||
raise ValueError("Set OPENROUTER_API_KEY or the deployment's api_key to list OpenRouter models")
|
||||
headers: Final = self.validate_environment(
|
||||
headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key
|
||||
)
|
||||
response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters
|
||||
url=_chat_catalog_url(self.get_api_base(api_base)), headers=headers, timeout=_CATALOG_TIMEOUT
|
||||
)
|
||||
response.raise_for_status()
|
||||
return [
|
||||
f"openrouter/{model.id}"
|
||||
for model in _Catalog.model_validate_json(response.content).data
|
||||
if not model.id.endswith(_BATCH_ONLY_SUFFIX)
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
# What is this?
|
||||
## Common checks for /v1/models and `/model/info`
|
||||
import asyncio
|
||||
import contextvars
|
||||
import copy
|
||||
import functools
|
||||
import itertools
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -30,6 +33,17 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
|
|||
|
||||
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
|
||||
|
||||
_MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing")
|
||||
|
||||
_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value})
|
||||
|
||||
_Listed: Final = TypeVar("_Listed")
|
||||
|
||||
|
||||
async def run_model_listing(listing: Callable[[], _Listed]) -> _Listed:
|
||||
context: Final = contextvars.copy_context()
|
||||
return await asyncio.get_running_loop().run_in_executor(_MODEL_LISTING_POOL, context.run, listing)
|
||||
|
||||
|
||||
def _check_wildcard_routing(model: str) -> bool:
|
||||
"""
|
||||
|
|
@ -209,6 +223,7 @@ def get_complete_model_list(
|
|||
include_model_access_groups: bool | None = False,
|
||||
only_model_access_groups: bool | None = False,
|
||||
team_id: str | None = None,
|
||||
expand_wildcard_routes: bool = True,
|
||||
) -> list[str]:
|
||||
"""Logic for returning complete model list for a given key + team pair"""
|
||||
|
||||
|
|
@ -254,11 +269,10 @@ def get_complete_model_list(
|
|||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
team_id=team_id,
|
||||
expand_wildcard_routes=expand_wildcard_routes,
|
||||
)
|
||||
|
||||
complete_model_list: Final = unique_models + all_wildcard_models
|
||||
|
||||
return complete_model_list
|
||||
return list(dict.fromkeys(unique_models + all_wildcard_models))
|
||||
|
||||
|
||||
def _hydrate_litellm_credential_name(
|
||||
|
|
@ -425,6 +439,15 @@ def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str |
|
|||
return functools.cache(wildcard_routed_models)
|
||||
|
||||
|
||||
def _deployed_by_name(llm_router: Router | None) -> Callable[[str, str | None], bool]:
|
||||
if llm_router is None:
|
||||
return lambda _model, _team_id: False
|
||||
registry: Final = llm_router.registry()
|
||||
deployed_names: Final = frozenset(registry.names(kind="group_set"))
|
||||
team_deployed_names: Final = frozenset(registry.index("team_model"))
|
||||
return lambda model, team_id: model in deployed_names or (team_id, model) in team_deployed_names
|
||||
|
||||
|
||||
def _routes_to_model(
|
||||
routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None
|
||||
) -> Callable[[str, str], bool]:
|
||||
|
|
@ -455,7 +478,11 @@ def get_known_models_from_wildcard(
|
|||
wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1)
|
||||
except ValueError: # safely fail
|
||||
return []
|
||||
deployment_pattern_naming_listed_ids: Final = _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
|
||||
deployment_pattern_naming_listed_ids: Final = (
|
||||
wildcard_model_to_expand
|
||||
if wildcard_model_to_expand != wildcard_model
|
||||
else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
|
||||
)
|
||||
|
||||
# Use provider from litellm_params when available, otherwise from wildcard prefix
|
||||
# (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router)
|
||||
|
|
@ -514,6 +541,27 @@ def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_pro
|
|||
return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}"
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=256)
|
||||
def _wildcard_name_regex(name_pattern: str) -> re.Pattern[str]:
|
||||
return re.compile(PatternMatchRouter().pattern_to_regex(name_pattern))
|
||||
|
||||
|
||||
def _routed_deployment_model(name_pattern: str, listed_name: str, deployment_model: str) -> str:
|
||||
name_match: Final = _wildcard_name_regex(name_pattern).match(listed_name)
|
||||
if name_match is None:
|
||||
return listed_name
|
||||
return PatternMatchRouter.set_deployment_model_name(
|
||||
matched_pattern=name_match, litellm_deployment_litellm_model=deployment_model
|
||||
)
|
||||
|
||||
|
||||
def _names_no_deployment_serves_by_name(
|
||||
deployment: Mapping[str, object], names: Sequence[str], deployed_by_name: Callable[[str, str | None], bool]
|
||||
) -> tuple[str, ...]:
|
||||
team_id: Final = _deployment_team_id(deployment)
|
||||
return tuple(name for name in names if not deployed_by_name(name, team_id))
|
||||
|
||||
|
||||
def expand_wildcard_deployments_for_model_info(
|
||||
deployments: list[dict[str, object]],
|
||||
llm_router: Router | None = None,
|
||||
|
|
@ -527,6 +575,7 @@ def expand_wildcard_deployments_for_model_info(
|
|||
"""
|
||||
expanded: Final[list[dict[str, object]]] = []
|
||||
routed_models: Final = _cached_wildcard_routing(llm_router)
|
||||
deployed_by_name: Final = _deployed_by_name(llm_router)
|
||||
for deployment in deployments:
|
||||
model_name = str(deployment.get("model_name") or "")
|
||||
raw_params = deployment.get("litellm_params")
|
||||
|
|
@ -560,22 +609,29 @@ def expand_wildcard_deployments_for_model_info(
|
|||
expanded.append(deployment)
|
||||
continue
|
||||
|
||||
for name in expanded_names:
|
||||
for name in _names_no_deployment_serves_by_name(deployment, expanded_names, deployed_by_name):
|
||||
row = copy.deepcopy(deployment)
|
||||
row["model_name"] = name
|
||||
params = row.get("litellm_params")
|
||||
if isinstance(params, dict):
|
||||
params["model"] = name
|
||||
params["model"] = _routed_deployment_model(model_name, name, litellm_model)
|
||||
expanded.append(row)
|
||||
|
||||
return expanded
|
||||
|
||||
|
||||
def _known_models_without_a_deployment(wildcard_model: str) -> Sequence[str]:
|
||||
if wildcard_model.partition("/")[0] in _PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT:
|
||||
return ()
|
||||
return get_known_models_from_wildcard(wildcard_model=wildcard_model, litellm_params=None)
|
||||
|
||||
|
||||
def _get_wildcard_models(
|
||||
unique_models: list[str],
|
||||
return_wildcard_routes: bool | None = False,
|
||||
llm_router: Router | None = None,
|
||||
team_id: str | None = None,
|
||||
expand_wildcard_routes: bool = True,
|
||||
) -> list[str]:
|
||||
models_to_remove: Final = set()
|
||||
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
|
||||
|
|
@ -586,6 +642,8 @@ def _get_wildcard_models(
|
|||
all_wildcard_models.append(model)
|
||||
|
||||
models_to_remove.add(model)
|
||||
if not expand_wildcard_routes:
|
||||
continue
|
||||
|
||||
model_list = llm_router.registry().deployments(model, team_id=team_id) if llm_router else None
|
||||
if model_list:
|
||||
|
|
@ -598,7 +656,7 @@ def _get_wildcard_models(
|
|||
)
|
||||
)
|
||||
else:
|
||||
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
|
||||
all_wildcard_models.extend(_known_models_without_a_deployment(model))
|
||||
|
||||
for model in models_to_remove:
|
||||
unique_models.remove(model)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import base64
|
||||
import itertools
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
|
|
@ -541,7 +542,8 @@ def get_team_provider_credentials(
|
|||
listings on the team's own provider account/key instead of a shared
|
||||
global one.
|
||||
2. Fallback: any deployment the caller is granted access to for this
|
||||
provider, expanding wildcard routes and the all-proxy-models sentinel.
|
||||
provider, matching wildcard grants against deployed model names without
|
||||
listing any provider's catalog, and expanding the all-proxy-models sentinel.
|
||||
Skipped when ``include_accessible_deployments`` is False.
|
||||
|
||||
Credential lookup is scoped to both the team's allowlist and the key's own
|
||||
|
|
@ -559,6 +561,7 @@ def get_team_provider_credentials(
|
|||
return None
|
||||
|
||||
from litellm.proxy._types import SpecialModelNames
|
||||
from litellm.proxy.auth.auth_checks import is_model_allowed_by_pattern
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models
|
||||
|
||||
team_id: Final = user_api_key_dict.team_id
|
||||
|
|
@ -645,28 +648,36 @@ def get_team_provider_credentials(
|
|||
grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models
|
||||
effective_team_models: Final = [] if grants_all_models else team_models
|
||||
|
||||
models_to_try: Final = list(
|
||||
dict.fromkeys(
|
||||
get_complete_model_list(
|
||||
key_models=list(key_model_allowlist),
|
||||
team_models=effective_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=True,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
team_id=team_id,
|
||||
)
|
||||
granted_models: Final = get_complete_model_list(
|
||||
key_models=list(key_model_allowlist),
|
||||
team_models=effective_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=True,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
team_id=team_id,
|
||||
expand_wildcard_routes=False,
|
||||
)
|
||||
deployments_granted_by_wildcard: Final = (
|
||||
model_name
|
||||
for model_name in proxy_model_list
|
||||
if any(
|
||||
is_model_allowed_by_pattern(model=model_name, allowed_model_pattern=granted) for granted in granted_models
|
||||
)
|
||||
)
|
||||
for model_name in models_to_try:
|
||||
credentials = _provider_credentials(model_name)
|
||||
if credentials is not None:
|
||||
return credentials
|
||||
|
||||
return None
|
||||
return next(
|
||||
(
|
||||
credentials
|
||||
for credentials in map(
|
||||
_provider_credentials, dict.fromkeys(itertools.chain(granted_models, deployments_granted_by_wildcard))
|
||||
)
|
||||
if credentials is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*")
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ from litellm.types.caching import RedisPipelineIncrementOperation
|
|||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
CountContextManagement,
|
||||
LlmProviders,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
TextCompletionResponse,
|
||||
|
|
@ -443,6 +444,7 @@ from litellm.proxy.auth.model_checks import (
|
|||
get_key_models,
|
||||
get_mcp_server_ids,
|
||||
get_team_models,
|
||||
run_model_listing,
|
||||
)
|
||||
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
|
||||
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports
|
||||
|
|
@ -12487,20 +12489,23 @@ async def model_list(
|
|||
# Get complete model list including wildcard routes if requested
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
all_models = get_complete_model_list(
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
proxy_model_list=[
|
||||
*proxy_model_list,
|
||||
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
|
||||
],
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=include_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups or False,
|
||||
only_model_access_groups=only_model_access_groups or False,
|
||||
all_models = await run_model_listing(
|
||||
partial(
|
||||
get_complete_model_list,
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
proxy_model_list=[
|
||||
*proxy_model_list,
|
||||
*(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
|
||||
],
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=include_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups or False,
|
||||
only_model_access_groups=only_model_access_groups or False,
|
||||
)
|
||||
)
|
||||
|
||||
expanded_undiscoverable_names: Final = undiscoverable_model_names(
|
||||
|
|
@ -14747,6 +14752,8 @@ from litellm.responses.litellm_completion_transformation.count_request import (
|
|||
with_provider_file_ids,
|
||||
)
|
||||
|
||||
_PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO: Final = frozenset({LlmProviders.OPENROUTER})
|
||||
|
||||
|
||||
def _with_client_oauth_token(
|
||||
counter: BaseTokenCounter | None, client_oauth_token: str | None
|
||||
|
|
@ -14782,14 +14789,13 @@ def _get_provider_token_counter(
|
|||
)
|
||||
|
||||
# Switch case pattern using existing get_provider_model_info
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
# Convert string provider to LlmProviders enum
|
||||
llm_provider_enum: Final = LlmProviders(custom_llm_provider)
|
||||
# Add more provider mappings as needed
|
||||
|
||||
if llm_provider_enum:
|
||||
if llm_provider_enum not in _PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO:
|
||||
provider_model_info: Final = ProviderConfigManager.get_provider_model_info(
|
||||
model=full_model, provider=llm_provider_enum
|
||||
)
|
||||
|
|
@ -17941,7 +17947,7 @@ async def model_info_v1(
|
|||
alias_models: Final = copy.deepcopy(registry.deployments(source="alias"))
|
||||
all_models.extend(alias_models)
|
||||
|
||||
all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router)
|
||||
all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router))
|
||||
|
||||
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -18279,14 +18285,17 @@ async def model_group_info(
|
|||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
)
|
||||
all_models_str: Final = (
|
||||
get_complete_model_list(
|
||||
key_models=(),
|
||||
team_models=(),
|
||||
proxy_model_list=llm_router.registry().names(),
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
|
||||
return_wildcard_routes=False,
|
||||
llm_router=llm_router,
|
||||
await run_model_listing(
|
||||
partial(
|
||||
get_complete_model_list,
|
||||
key_models=(),
|
||||
team_models=(),
|
||||
proxy_model_list=llm_router.registry().names(),
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
|
||||
return_wildcard_routes=False,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
)
|
||||
if is_proxy_admin
|
||||
else await get_available_models_for_user(
|
||||
|
|
|
|||
|
|
@ -9500,7 +9500,7 @@ async def _agent_access_group_visible_models(
|
|||
) -> frozenset[str] | None:
|
||||
"""Models an agent key may still list once its attached access groups cap it, ``None`` when
|
||||
nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on."""
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_listing
|
||||
|
||||
if not user_api_key_dict.agent_id:
|
||||
return None
|
||||
|
|
@ -9521,17 +9521,20 @@ async def _agent_access_group_visible_models(
|
|||
if not granted:
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
get_complete_model_list(
|
||||
key_models=granted,
|
||||
team_models=(),
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
team_id=team_id,
|
||||
await run_model_listing(
|
||||
partial(
|
||||
get_complete_model_list,
|
||||
key_models=granted,
|
||||
team_models=(),
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
team_id=team_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -9576,6 +9579,7 @@ async def get_available_models_for_user(
|
|||
get_complete_model_list,
|
||||
get_key_models,
|
||||
get_team_models,
|
||||
run_model_listing,
|
||||
)
|
||||
|
||||
# Get proxy model list and access groups
|
||||
|
|
@ -9656,18 +9660,21 @@ async def get_available_models_for_user(
|
|||
)
|
||||
|
||||
# Get complete model list
|
||||
complete_models: Final = get_complete_model_list(
|
||||
key_models=granted_key_models,
|
||||
team_models=granted_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
only_model_access_groups=only_model_access_groups,
|
||||
team_id=effective_team_id,
|
||||
complete_models: Final = await run_model_listing(
|
||||
partial(
|
||||
get_complete_model_list,
|
||||
key_models=granted_key_models,
|
||||
team_models=granted_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
only_model_access_groups=only_model_access_groups,
|
||||
team_id=effective_team_id,
|
||||
)
|
||||
)
|
||||
all_models: Final = (
|
||||
list(
|
||||
|
|
|
|||
|
|
@ -279,7 +279,9 @@ class PatternMatchRouter:
|
|||
PatternMatchRouter.set_deployment_model_name(
|
||||
matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"]
|
||||
)
|
||||
for deployment in self.patterns[pattern]
|
||||
for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts
|
||||
pattern, ()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8724,11 +8724,13 @@ class AvailableModelsCache(InMemoryCache):
|
|||
|
||||
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
|
||||
|
||||
result: Final = cast(list[str] | None, self.get_cache(cache_key))
|
||||
result: Final = self._cached(cache_key)
|
||||
|
||||
if result is not None:
|
||||
return copy.deepcopy(result)
|
||||
return result
|
||||
if isinstance(result, Sequence):
|
||||
return list(result)
|
||||
if self._cached(_failed_listing_key(cache_key)) is not None:
|
||||
return []
|
||||
return None
|
||||
|
||||
def set_cached_model_info(
|
||||
self,
|
||||
|
|
@ -8738,12 +8740,46 @@ class AvailableModelsCache(InMemoryCache):
|
|||
):
|
||||
"""Set cached model info"""
|
||||
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
|
||||
self.set_cache(cache_key, copy.deepcopy(available_models))
|
||||
self._store(cache_key, copy.deepcopy(available_models))
|
||||
|
||||
def set_failed_model_listing(
|
||||
self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None, ttl_seconds: float
|
||||
) -> None:
|
||||
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
|
||||
self._store(_failed_listing_key(cache_key), True, ttl=ttl_seconds)
|
||||
|
||||
def listing_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock:
|
||||
return _provider_listing_lock(self._get_cache_key(custom_llm_provider, litellm_params))
|
||||
|
||||
def _cached(self, key: str) -> Sequence[str] | bool | None:
|
||||
return cast(Sequence[str] | bool | None, self.get_cache(key))
|
||||
|
||||
def _store(self, key: str, value: object, ttl: float | None = None) -> None:
|
||||
self.set_cache(key, value, ttl=ttl)
|
||||
|
||||
|
||||
def _failed_listing_key(cache_key: str) -> str:
|
||||
return f"{cache_key}:failed"
|
||||
|
||||
|
||||
_PROVIDER_LISTING_LOCK_GUARD: Final = threading.Lock()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _cached_provider_listing_lock(cache_key: str) -> threading.RLock:
|
||||
return threading.RLock()
|
||||
|
||||
|
||||
def _provider_listing_lock(cache_key: str) -> threading.RLock:
|
||||
with _PROVIDER_LISTING_LOCK_GUARD:
|
||||
return _cached_provider_listing_lock(cache_key)
|
||||
|
||||
|
||||
# Global cache instance
|
||||
_model_cache: Final = AvailableModelsCache()
|
||||
|
||||
_FAILED_PROVIDER_LISTING_TTL_SECONDS: Final = 30
|
||||
|
||||
|
||||
def _infer_valid_provider_from_env_vars(
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
@ -8791,20 +8827,27 @@ def _get_valid_models_from_provider_api(
|
|||
custom_llm_provider: str,
|
||||
litellm_params: LiteLLM_Params | None = None,
|
||||
) -> list[str]:
|
||||
try:
|
||||
cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
|
||||
with _model_cache.listing_lock(custom_llm_provider, litellm_params):
|
||||
try:
|
||||
cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params)
|
||||
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
models: Final = provider_config.discover_models(
|
||||
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
|
||||
)
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
models: Final = provider_config.discover_models(
|
||||
litellm_params=_provider_listing_params(custom_llm_provider, litellm_params)
|
||||
)
|
||||
|
||||
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
return models
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
return []
|
||||
_model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
|
||||
return models
|
||||
except httpx.TimeoutException as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
_model_cache.set_failed_model_listing(
|
||||
custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS
|
||||
)
|
||||
return []
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def bundled_provider_models(provider: str) -> tuple[str, ...]:
|
||||
|
|
@ -10470,6 +10513,10 @@ class ProviderConfigManager:
|
|||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
|
||||
return AzureFoundryModelInfo(model=model)
|
||||
elif LlmProviders.OPENROUTER == provider:
|
||||
from litellm.llms.openrouter.common_utils import OpenRouterModelInfo
|
||||
|
||||
return OpenRouterModelInfo()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
159
tests/unit/llms/openrouter/test_openrouter_common_utils.py
Normal file
159
tests/unit/llms/openrouter/test_openrouter_common_utils.py
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openrouter.common_utils import OpenRouterModelInfo
|
||||
|
||||
DEFAULT_BASE: Final = "https://openrouter.ai/api/v1"
|
||||
KEY_SOURCES: Final = ("api_key", "litellm.api_key", "litellm.openrouter_key", "OPENROUTER_API_KEY", "OR_API_KEY")
|
||||
BASE_SOURCES: Final = ("api_base", "litellm.api_base", "OPENROUTER_API_BASE")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_ambient_openrouter_settings(monkeypatch: pytest.MonkeyPatch):
|
||||
for env_var in ("OPENROUTER_API_BASE", "OPENROUTER_API_KEY", "OR_API_KEY"):
|
||||
monkeypatch.delenv(env_var, raising=False)
|
||||
monkeypatch.setattr(litellm, "api_key", None)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setattr(litellm, "openrouter_key", None)
|
||||
|
||||
|
||||
def _catalog(*model_ids: str) -> httpx.Response:
|
||||
return httpx.Response(200, json={"data": [{"id": model_id, "name": model_id} for model_id in model_ids]})
|
||||
|
||||
|
||||
def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None:
|
||||
if source.startswith("litellm."):
|
||||
monkeypatch.setattr(litellm, source.removeprefix("litellm."), value)
|
||||
else:
|
||||
monkeypatch.setenv(source, value)
|
||||
|
||||
|
||||
def test_get_models_lists_only_the_chat_catalog_under_openrouter_keeping_the_org_namespace():
|
||||
with respx.mock() as respx_mock:
|
||||
respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(
|
||||
return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5")
|
||||
)
|
||||
|
||||
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
|
||||
"openrouter/openai/gpt-5.4-mini",
|
||||
"openrouter/anthropic/claude-haiku-5-5",
|
||||
]
|
||||
assert [call.request.url.path for call in respx_mock.calls] == ["/api/v1/models/user"]
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_models_drops_batch_only_ids_and_keeps_every_other_variant():
|
||||
respx.get(f"{DEFAULT_BASE}/models/user").mock(
|
||||
return_value=_catalog("openai/gpt-5.4-mini:batch", "openai/gpt-5.4-mini", "qwen/qwen3-coder:free")
|
||||
)
|
||||
|
||||
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
|
||||
"openrouter/openai/gpt-5.4-mini",
|
||||
"openrouter/qwen/qwen3-coder:free",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
[
|
||||
"https://gateway.example/openrouter/api/v1/",
|
||||
"https://gateway.example/openrouter/api/v1/chat/completions",
|
||||
"https://gateway.example/openrouter/api/v1/chat/completions/",
|
||||
],
|
||||
)
|
||||
@respx.mock
|
||||
def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(api_base: str):
|
||||
chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock(
|
||||
return_value=_catalog("openai/gpt-5.4-mini")
|
||||
)
|
||||
|
||||
assert OpenRouterModelInfo().get_models(api_key="sk-or-deployment", api_base=api_base) == [
|
||||
"openrouter/openai/gpt-5.4-mini"
|
||||
]
|
||||
assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-deployment"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES)
|
||||
@respx.mock
|
||||
def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order(
|
||||
monkeypatch: pytest.MonkeyPatch, winner: int
|
||||
):
|
||||
for source in KEY_SOURCES[winner + 1 :]:
|
||||
_configure(monkeypatch, source, f"sk-or-from-{source}")
|
||||
if winner > 0:
|
||||
_configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner")
|
||||
chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder"))
|
||||
|
||||
assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [
|
||||
"openrouter/qwen/qwen3-coder"
|
||||
]
|
||||
assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-winner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("winner", range(len(BASE_SOURCES) + 1), ids=(*BASE_SOURCES, "default"))
|
||||
@respx.mock
|
||||
def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order(
|
||||
monkeypatch: pytest.MonkeyPatch, winner: int
|
||||
):
|
||||
for source in BASE_SOURCES[winner + 1 :]:
|
||||
_configure(monkeypatch, source, f"https://{source.lower().replace('.', '-')}.example/api/v1")
|
||||
if 0 < winner < len(BASE_SOURCES):
|
||||
_configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1")
|
||||
expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE
|
||||
respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder"))
|
||||
|
||||
assert OpenRouterModelInfo().get_models(
|
||||
api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None
|
||||
) == ["openrouter/qwen/qwen3-coder"]
|
||||
|
||||
|
||||
def test_get_models_without_any_key_raises_before_calling_openrouter():
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
chat_route: Final = respx_mock.get(f"{DEFAULT_BASE}/models/user").mock(
|
||||
return_value=_catalog("openai/gpt-5.4-mini")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="OPENROUTER_API_KEY"):
|
||||
OpenRouterModelInfo().get_models()
|
||||
|
||||
assert not chat_route.called
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_models_raises_when_the_catalog_answers_an_error():
|
||||
respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=httpx.Response(401, json={"error": {"code": 401}}))
|
||||
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
OpenRouterModelInfo().get_models(api_key="sk-or-revoked")
|
||||
|
||||
|
||||
def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-env")
|
||||
headers: Final = {"X-Title": "my-app"}
|
||||
|
||||
assert OpenRouterModelInfo().validate_environment(
|
||||
headers=headers, model="openrouter/openai/gpt-5.4-mini", messages=(), optional_params={}, litellm_params={}
|
||||
) == {"X-Title": "my-app", "Authorization": "Bearer sk-or-env"}
|
||||
assert headers == {"X-Title": "my-app"}
|
||||
|
||||
|
||||
def test_validate_environment_without_any_key_sends_no_authorization():
|
||||
assert OpenRouterModelInfo().validate_environment(
|
||||
headers={"X-Title": "my-app"},
|
||||
model="openrouter/openai/gpt-5.4-mini",
|
||||
messages=(),
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
) == {"X-Title": "my-app"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[("openrouter/openai/gpt-5.4-mini", "openai/gpt-5.4-mini"), ("openai/gpt-5.4-mini", "openai/gpt-5.4-mini")],
|
||||
)
|
||||
def test_get_base_model_drops_only_the_openrouter_prefix(model: str, base_model: str):
|
||||
assert OpenRouterModelInfo.get_base_model(model) == base_model
|
||||
|
|
@ -7,6 +7,8 @@ import httpx
|
|||
import pytest
|
||||
import respx
|
||||
|
||||
from litellm import Router
|
||||
|
||||
|
||||
def test_get_team_models_for_all_models_and_team_only_models():
|
||||
from litellm.proxy.auth.model_checks import get_team_models
|
||||
|
|
@ -1261,6 +1263,223 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it
|
|||
assert get_known_models_from_wildcard("bedrock/anthropic.*", deployment) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("wildcard", "deployment_model", "listed_ids"),
|
||||
[
|
||||
(
|
||||
"openrouter/*",
|
||||
"openrouter/*",
|
||||
["openrouter/openai/gpt-5.4-mini", "openrouter/anthropic/claude-haiku-5-5"],
|
||||
),
|
||||
("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]),
|
||||
("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]),
|
||||
("*", "openrouter/*", ["openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5"]),
|
||||
],
|
||||
)
|
||||
@respx.mock
|
||||
def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace(
|
||||
monkeypatch: pytest.MonkeyPatch, wildcard: str, deployment_model: str, listed_ids: list[str]
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
|
||||
respx.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
return_value=httpx.Response(
|
||||
200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]}
|
||||
)
|
||||
)
|
||||
|
||||
assert (
|
||||
get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}"))
|
||||
== listed_ids
|
||||
)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_no_environment_catalog(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-environment-only")
|
||||
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[{"model_name": "or-team/*", "litellm_params": {"model": "openrouter/*", "api_key": "sk-or-team"}}]
|
||||
)
|
||||
|
||||
listed: Final = get_complete_model_list(
|
||||
key_models=["openrouter/*"],
|
||||
team_models=[],
|
||||
proxy_model_list=router.model_names,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
assert (listed, catalog.call_count) == ([], 0)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _openai_catalog_behind_a_catch_all(
|
||||
monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str]
|
||||
) -> Generator[Router, None, None]:
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
respx_mock.get("https://api.openai.com/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "gpt-4o-mini"}]})
|
||||
)
|
||||
yield Router(
|
||||
model_list=[
|
||||
{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-openai-catch-all"}},
|
||||
{
|
||||
"model_name": concrete_name,
|
||||
"litellm_params": {"model": "openrouter/openai/gpt-4o", "api_key": "sk-or-gpt-4o"},
|
||||
"model_info": dict(concrete_model_info),
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("concrete_name", "concrete_model_info", "team_id"),
|
||||
[
|
||||
("gpt-4o", {}, None),
|
||||
("model_name_team-x_gpt-4o", {"team_id": "team-x", "team_public_model_name": "gpt-4o"}, "team-x"),
|
||||
],
|
||||
)
|
||||
def test_catch_all_wildcard_lists_a_name_a_concrete_deployment_serves_once(
|
||||
monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str], team_id: str | None
|
||||
) -> None:
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
with _openai_catalog_behind_a_catch_all(monkeypatch, concrete_name, concrete_model_info) as router:
|
||||
listed: Final = get_complete_model_list(
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
proxy_model_list=router.get_model_names(team_id=team_id),
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
llm_router=router,
|
||||
team_id=team_id,
|
||||
)
|
||||
|
||||
assert sorted(listed) == ["gpt-4o", "gpt-4o-mini"]
|
||||
|
||||
|
||||
def test_catch_all_wildcard_model_info_keeps_only_the_concrete_row_for_a_deployed_name(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info
|
||||
|
||||
with _openai_catalog_behind_a_catch_all(monkeypatch, "gpt-4o", {}) as router:
|
||||
rows: Final = expand_wildcard_deployments_for_model_info(
|
||||
[dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router
|
||||
)
|
||||
|
||||
assert sorted((row["model_name"], row["litellm_params"]["model"]) for row in rows) == [
|
||||
("gpt-4o", "openrouter/openai/gpt-4o"),
|
||||
("gpt-4o-mini", "openai/gpt-4o-mini"),
|
||||
]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _openrouter_catalog_beside_a_named_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
wildcard_name: str,
|
||||
named: str,
|
||||
catalog: tuple[str, ...] = ("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5"),
|
||||
wildcard_key: str | None = None,
|
||||
) -> Generator[Router, None, None]:
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
respx_mock.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": catalog_id} for catalog_id in catalog]})
|
||||
)
|
||||
yield Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": wildcard_name,
|
||||
"litellm_params": {
|
||||
"model": "openrouter/*",
|
||||
"api_key": wildcard_key or f"sk-or-wildcard-{wildcard_name}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": named,
|
||||
"litellm_params": {"model": "openrouter/openai/gpt-5.4-mini", "api_key": "sk-or-named"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("wildcard_name", "named"),
|
||||
[("openrouter/*", "openrouter/openai/gpt-5.4-mini"), ("or-team/*", "or-team/openai/gpt-5.4-mini")],
|
||||
)
|
||||
def test_provider_wildcard_lists_a_name_a_named_deployment_serves_once(
|
||||
monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str
|
||||
) -> None:
|
||||
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
|
||||
|
||||
with _openrouter_catalog_beside_a_named_deployment(monkeypatch, wildcard_name, named) as router:
|
||||
listed: Final = get_complete_model_list(
|
||||
key_models=[],
|
||||
team_models=[],
|
||||
proxy_model_list=list(router.model_names),
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
llm_router=router,
|
||||
)
|
||||
rows: Final = expand_wildcard_deployments_for_model_info(
|
||||
[dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router
|
||||
)
|
||||
|
||||
vendor_prefix: Final = wildcard_name.removesuffix("*")
|
||||
assert sorted(listed) == [f"{vendor_prefix}anthropic/claude-haiku-5-5", named]
|
||||
assert sorted((row["model_name"], row["litellm_params"]["api_key"]) for row in rows) == [
|
||||
(f"{vendor_prefix}anthropic/claude-haiku-5-5", f"sk-or-wildcard-{wildcard_name}"),
|
||||
(named, "sk-or-named"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("wildcard_name", "named"),
|
||||
[("openrouter/*", "openrouter/openai/gpt-5.4-mini"), ("or-team/*", "or-team/openai/gpt-5.4-mini")],
|
||||
)
|
||||
def test_provider_wildcard_whose_whole_catalog_is_deployed_by_name_shows_only_the_named_row(
|
||||
monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str
|
||||
) -> None:
|
||||
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info
|
||||
|
||||
with _openrouter_catalog_beside_a_named_deployment(
|
||||
monkeypatch,
|
||||
wildcard_name,
|
||||
named,
|
||||
catalog=("openai/gpt-5.4-mini",),
|
||||
wildcard_key=f"sk-or-one-model-{wildcard_name}",
|
||||
) as router:
|
||||
rows: Final = expand_wildcard_deployments_for_model_info(
|
||||
[dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router
|
||||
)
|
||||
|
||||
assert [(row["model_name"], row["litellm_params"]["api_key"]) for row in rows] == [(named, "sk-or-named")]
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
|
|
@ -3,10 +3,13 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
|
@ -852,3 +855,112 @@ def test_a_team_keeps_its_first_config_file_openai_deployment_over_older_db_ones
|
|||
|
||||
assert credentials is not None
|
||||
assert credentials["api_key"] == "deployment-z-key"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("custom_llm_provider", "openrouter_key", "api_key"),
|
||||
[("openai", "sk-or-files-openai", "openai-key"), ("openrouter", "sk-or-files-openrouter", "sk-or-files-openrouter")],
|
||||
)
|
||||
def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_catalog(
|
||||
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, openrouter_key: str, api_key: str
|
||||
):
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openrouter/*",
|
||||
"litellm_params": {
|
||||
"model": "openrouter/*",
|
||||
"api_key": openrouter_key,
|
||||
"api_base": "https://openrouter.ai/api/v1",
|
||||
},
|
||||
},
|
||||
{"model_name": "gpt-5.4-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "openai-key"}},
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
catalog_route: Final = respx_mock.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
|
||||
)
|
||||
credentials: Final = get_team_provider_credentials(
|
||||
llm_router=router,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user"),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
include_accessible_deployments=True,
|
||||
)
|
||||
|
||||
assert credentials is not None
|
||||
assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key)
|
||||
assert catalog_route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key_models", "deployments", "custom_llm_provider", "api_key"),
|
||||
[
|
||||
(
|
||||
["anthropic/*"],
|
||||
(("anthropic/claude-*", "anthropic/claude-*", "anthropic-narrow-key"),),
|
||||
"anthropic",
|
||||
"anthropic-narrow-key",
|
||||
),
|
||||
(["anthropic/claude-3*"], (("anthropic/*", "anthropic/*", "anthropic-wide-key"),), "anthropic", "anthropic-wide-key"),
|
||||
(["openrouter/*"], (("openrouter/openai/*", "openrouter/openai/*", "sk-or-narrow"),), "openrouter", "sk-or-narrow"),
|
||||
(
|
||||
["team/*"],
|
||||
(("team/*", "openrouter/*", "sk-or-team"), ("team/openai/*", "openai/*", "openai-team-key")),
|
||||
"openai",
|
||||
"openai-team-key",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_key_wildcard_finds_the_deployment_it_grants_without_listing_any_provider_catalog(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
key_models: list[str],
|
||||
deployments: tuple[tuple[str, str, str], ...],
|
||||
custom_llm_provider: str,
|
||||
api_key: str,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": model_name, "litellm_params": {"model": model, "api_key": deployment_key}}
|
||||
for model_name, model, deployment_key in deployments
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
catalog_requests: Final = respx_mock.route().mock(return_value=httpx.Response(200, json={"data": []}))
|
||||
credentials: Final = get_team_provider_credentials(
|
||||
llm_router=router,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=key_models),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
include_accessible_deployments=True,
|
||||
)
|
||||
|
||||
assert credentials is not None
|
||||
assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key)
|
||||
assert catalog_requests.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployed_name", "api_key"), [("gpt-4o-mini", None), ("openai/gpt-4o-mini", "openai-deployment-key")]
|
||||
)
|
||||
def test_a_provider_wildcard_grant_finds_only_deployments_named_under_it(deployed_name: str, api_key: str | None):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": deployed_name,
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "openai-deployment-key"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
credentials: Final = get_team_provider_credentials(
|
||||
llm_router=router,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["openai/*"]),
|
||||
custom_llm_provider="openai",
|
||||
include_accessible_deployments=True,
|
||||
)
|
||||
|
||||
assert (None if credentials is None else credentials["api_key"]) == api_key
|
||||
|
|
|
|||
|
|
@ -550,15 +550,12 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke
|
|||
|
||||
deployment = {
|
||||
"model_name": "*",
|
||||
"litellm_params": {"model": "openai/*"},
|
||||
"litellm_params": {"model": "openai/*", "api_key": "sk-star"},
|
||||
}
|
||||
router = route_entry_points_to_stubs(MagicMock())
|
||||
router.get_model_access_groups = MagicMock(return_value={})
|
||||
router.get_model_names = MagicMock(return_value=["*"])
|
||||
router.get_model_list = MagicMock(return_value=[deployment])
|
||||
router = litellm.Router(model_list=[deployment])
|
||||
monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models)
|
||||
|
||||
expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info([deployment])
|
||||
expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info(router.get_model_list(), router)
|
||||
allowed_model_names = await proxy_server._get_v1_model_info_allowed_model_names_with_access_groups(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="sk-test",
|
||||
|
|
@ -575,7 +572,8 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke
|
|||
allowed_model_names=allowed_model_names,
|
||||
)
|
||||
|
||||
assert [model["model_name"] for model in result] == ["openai/gpt-4o"]
|
||||
assert [(model["model_name"], model["litellm_params"]["model"]) for model in result] == [("gpt-4o", "openai/gpt-4o")]
|
||||
assert router.registry().names("gpt-4o", kind="wildcard") == frozenset({"openai/gpt-4o"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -9,10 +9,16 @@ Pins (PR2):
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -541,6 +547,73 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat
|
|||
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
|
||||
|
||||
|
||||
OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CatalogHeldUntilTheLoopRuns:
|
||||
requested: threading.Event = field(default_factory=threading.Event)
|
||||
released: threading.Event = field(default_factory=threading.Event)
|
||||
|
||||
def answer(self, _request: httpx.Request) -> httpx.Response:
|
||||
self.requested.set()
|
||||
if not self.released.wait(timeout=30):
|
||||
return httpx.Response(504)
|
||||
return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
|
||||
|
||||
async def release_from_the_loop(self) -> None:
|
||||
assert await asyncio.to_thread(self.requested.wait, 30)
|
||||
self.released.set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope", [None, "expand"])
|
||||
def test_v1_models_fetches_a_live_openrouter_catalog_without_holding_the_event_loop_or_its_default_thread_pool(
|
||||
monkeypatch: pytest.MonkeyPatch, scope: str | None
|
||||
):
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openrouter/*",
|
||||
"litellm_params": {
|
||||
"model": "openrouter/*",
|
||||
"api_key": f"sk-or-v1-models-off-loop-{scope}",
|
||||
"api_base": OPENROUTER_BASE,
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
catalog: Final = _CatalogHeldUntilTheLoopRuns()
|
||||
|
||||
async def listed_ids_on_a_one_thread_default_pool() -> list[str]:
|
||||
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
|
||||
listing: Final = asyncio.ensure_future(
|
||||
proxy_server.model_list(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="sk-test-key", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
|
||||
),
|
||||
scope=scope,
|
||||
)
|
||||
)
|
||||
await catalog.release_from_the_loop()
|
||||
response: Final = await listing
|
||||
return [str(model["id"]) for model in response["data"]]
|
||||
|
||||
with respx.mock() as respx_mock:
|
||||
catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer)
|
||||
listed_ids: Final = asyncio.run(listed_ids_on_a_one_thread_default_pool())
|
||||
|
||||
assert catalog_route.calls.last.response.status_code == 200
|
||||
assert listed_ids == ["openrouter/openai/gpt-5.4-mini"]
|
||||
|
||||
|
||||
def _priced_jev_models() -> frozenset[str]:
|
||||
return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/"))
|
||||
|
||||
|
|
|
|||
|
|
@ -179,6 +179,39 @@ async def test_token_counting_model_not_in_model_list():
|
|||
assert response.model_used == "special-alias"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_asking_the_provider_endpoint_leaves_an_openrouter_deployments_count_unchanged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "or-gpt",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"custom_llm_provider": "openrouter",
|
||||
"api_key": "sk-or-token-count",
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
request: Final = TokenCountRequest(
|
||||
model="or-gpt", messages=[{"role": "user", "content": "Count these tokens, ünïcödé 你好 🙂"}]
|
||||
)
|
||||
|
||||
via_provider_endpoint: Final = await token_counter(request=request, call_endpoint=True)
|
||||
counted_locally: Final = await token_counter(request=request, call_endpoint=False)
|
||||
|
||||
assert (via_provider_endpoint.total_tokens, via_provider_endpoint.model_used) == (
|
||||
counted_locally.total_tokens,
|
||||
counted_locally.model_used,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gpt_token_counting():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,10 +1,17 @@
|
|||
import asyncio
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from litellm import ModelResponse, Router
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import (
|
||||
create_model_info_response,
|
||||
|
|
@ -650,3 +657,61 @@ async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups(
|
|||
resolve_agent_ceiling=_agent_ceiling(None),
|
||||
)
|
||||
assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"])
|
||||
|
||||
|
||||
OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CatalogHeldUntilTheLoopRuns:
|
||||
requested: threading.Event = field(default_factory=threading.Event)
|
||||
released: threading.Event = field(default_factory=threading.Event)
|
||||
|
||||
def answer(self, _request: httpx.Request) -> httpx.Response:
|
||||
self.requested.set()
|
||||
if not self.released.wait(timeout=30):
|
||||
return httpx.Response(504)
|
||||
return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]})
|
||||
|
||||
async def release_from_the_loop(self) -> None:
|
||||
assert await asyncio.to_thread(self.requested.wait, 30)
|
||||
self.released.set()
|
||||
|
||||
|
||||
def test_get_available_models_for_user_fetches_a_live_catalog_without_holding_the_event_loop_or_its_default_pool(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
catalog: Final = _CatalogHeldUntilTheLoopRuns()
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openrouter/*",
|
||||
"litellm_params": {
|
||||
"model": "openrouter/*",
|
||||
"api_key": "sk-or-available-models-off-loop",
|
||||
"api_base": OPENROUTER_BASE,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
async def list_models_on_a_one_thread_default_pool() -> list[str]:
|
||||
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
|
||||
listing: Final = asyncio.ensure_future(
|
||||
get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"),
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
)
|
||||
)
|
||||
await catalog.release_from_the_loop()
|
||||
return await listing
|
||||
|
||||
with respx.mock() as respx_mock:
|
||||
catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer)
|
||||
models: Final = asyncio.run(list_models_on_a_one_thread_default_pool())
|
||||
|
||||
assert catalog_route.calls.last.response.status_code == 200
|
||||
assert models == ["openrouter/openai/gpt-5.4-mini"]
|
||||
|
|
|
|||
|
|
@ -9269,6 +9269,77 @@ def reset_mock_cache() -> None:
|
|||
_model_cache.flush_cache()
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_listing(reset_mock_cache: None) -> None:
|
||||
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
side_effect=httpx.ReadTimeout("catalog hung")
|
||||
)
|
||||
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage")
|
||||
|
||||
listings: Final = tuple(
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
|
||||
for _ in range(2)
|
||||
)
|
||||
|
||||
assert (listings, catalog.call_count) == (([], []), 1)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None:
|
||||
respx.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
side_effect=(httpx.Response(503), httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}))
|
||||
)
|
||||
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-recovers")
|
||||
|
||||
listings: Final = tuple(
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
|
||||
for _ in range(2)
|
||||
)
|
||||
|
||||
assert listings == ([], ["openrouter/anthropic/claude-haiku-5-5"])
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it(
|
||||
reset_mock_cache: None,
|
||||
) -> None:
|
||||
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings")
|
||||
first_request: Final = iter((True, False))
|
||||
|
||||
def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response:
|
||||
if next(first_request):
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
|
||||
raise httpx.ReadTimeout("catalog hung", request=request)
|
||||
return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
|
||||
|
||||
respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing)
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
|
||||
|
||||
assert get_valid_models(
|
||||
check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment
|
||||
) == ["openrouter/anthropic/claude-haiku-5-5"]
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_asks_a_provider_once_when_many_listings_start_together(reset_mock_cache: None) -> None:
|
||||
def slow_catalog(request: httpx.Request) -> httpx.Response:
|
||||
threading.Event().wait(0.2)
|
||||
return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})
|
||||
|
||||
catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=slow_catalog)
|
||||
deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-many-listings")
|
||||
|
||||
def list_models(_: int) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment)
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
listings: Final = frozenset(pool.map(list_models, range(8)))
|
||||
|
||||
assert (listings, catalog.call_count) == (frozenset({("openrouter/anthropic/claude-haiku-5-5",)}), 1)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue