From 9176e4df180a49b07d605670be31497a03329baa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:31:06 -0700 Subject: [PATCH] fix(openrouter): show published limits on the proxy model-info routes An OpenRouter model with no cost-map row showed a null context window and output cap on /v1/model/info and /model_group/info, so lite pi fell back to 128k. The model-info refresh now reads OpenRouter's /models/user catalog, keeps one shared catalog per wildcard deployment, and looks up each concrete model on read, with no new call on the request path. Operator model_info still wins, and catalog values win over a cost-map row. Wildcard rows on /v1/models show the same limits, since the lite OpenCode and Codex model sync reads them there --- litellm/llms/openai_like/model_info.py | 90 +++-- litellm/llms/openrouter/common_utils.py | 59 ++- litellm/proxy/proxy_server.py | 2 +- litellm/router.py | 164 +++++++-- litellm/router_api.py | 9 +- litellm/router_deprecated.py | 4 +- .../router_utils/pattern_match_deployments.py | 23 +- litellm/types/router.py | 1 + .../providers/_cache_control_marks_support.py | 2 + .../test_openrouter_catalog_limits_wire.py | 109 ++++++ .../unit/llms/openai_like/test_model_info.py | 16 + .../test_openrouter_common_utils.py | 113 +++++- .../proxy_server/test_routes_model_info.py | 88 +++++ tests/unit/proxy/router_entry_point_stubs.py | 2 +- .../test_router/test_router_entry_points.py | 2 +- .../unit/test_router_model_cost_isolation.py | 347 ++++++++++++++++++ 16 files changed, 949 insertions(+), 82 deletions(-) create mode 100644 tests/integration/routing/test_openrouter_catalog_limits_wire.py diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index e3b2dc427c4..9a487994d79 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -1,6 +1,7 @@ import hashlib import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import Annotated, Final, TypeAlias @@ -17,23 +18,24 @@ MODEL_INFO_REFRESH_SECONDS: Final = 300 MODEL_INFO_REFRESH_CONCURRENCY: Final = 8 MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset({"hosted_vllm", "openai", "text-completion-openai", "openai_like"}) _EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({}) +_DISCOVERY_TIMEOUT: Final = httpx.Timeout(5.0) def _positive_limit(value: object) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None -_TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] +TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] class _ModelCard(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) id: str - max_model_len: _TokenLimit = None - context_length: _TokenLimit = None - max_input_tokens: _TokenLimit = None - max_output_tokens: _TokenLimit = None + max_model_len: TokenLimit = None + context_length: TokenLimit = None + max_input_tokens: TokenLimit = None + max_output_tokens: TokenLimit = None def token_limits(self) -> Mapping[str, int]: context: Final = self.max_model_len or self.context_length @@ -58,6 +60,51 @@ class _ModelList(LiteLLMBaseModel): data: tuple[_ModelCard, ...] = () +@dataclass(frozen=True, slots=True) +class _CachedModelLimits: + by_model: Mapping[str, Mapping[str, int]] + + +def _limits_of_first_card_per_id(content: bytes) -> Mapping[str, Mapping[str, int]]: + cards: Final = _ModelList.model_validate_json(content).data + return MappingProxyType({card.id: card.token_limits() for card in reversed(cards)}) + + +async def get_cached_model_limits( + *, + url: str, + headers: Mapping[str, str], + parse: Callable[[bytes], Mapping[str, Mapping[str, int]]], + client: AsyncHTTPHandler, + cache: InMemoryCache, + timeout: httpx.Timeout = _DISCOVERY_TIMEOUT, +) -> Mapping[str, Mapping[str, int]]: + cache_key: Final = ( + "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() + ) + cached: Final[object] = cache.get_cache(cache_key) + if isinstance(cached, _CachedModelLimits): + return cached.by_model + + try: + response: Final = await client.get( + url=url, + headers=dict(headers), + timeout=timeout, + follow_redirects=False, + max_response_bytes=2 * 1024 * 1024, + ) + response.raise_for_status() + limits: Final = parse(response.content) + except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh + verbose_logger.debug("Could not discover upstream model token limits") + cache.set_cache(cache_key, _CachedModelLimits(MappingProxyType({})), ttl=60) + return MappingProxyType({}) + + cache.set_cache(cache_key, _CachedModelLimits(limits), ttl=MODEL_INFO_REFRESH_SECONDS) + return limits + + async def get_openai_compatible_model_info( *, model: str, @@ -66,28 +113,11 @@ async def get_openai_compatible_model_info( client: AsyncHTTPHandler, cache: InMemoryCache, ) -> Mapping[str, int]: - url: Final = add_path_to_api_base(api_base, "/v1/models") - cache_key: Final = ( - "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() + limits: Final = await get_cached_model_limits( + url=add_path_to_api_base(api_base, "/v1/models"), + headers=headers, + parse=_limits_of_first_card_per_id, + client=client, + cache=cache, ) - cached: Final[object] = cache.get_cache(cache_key) - if isinstance(cached, _ModelList): - return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS) - - try: - response: Final = await client.get( - url=url, - headers=dict(headers), - timeout=httpx.Timeout(5.0), - follow_redirects=False, - max_response_bytes=2 * 1024 * 1024, - ) - response.raise_for_status() - models: Final = _ModelList.model_validate_json(response.content) - except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh - verbose_logger.debug("Could not discover upstream model token limits") - cache.set_cache(cache_key, _ModelList(), ttl=60) - return _EMPTY_LIMITS - - cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS) - return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS) + return limits.get(model, _EMPTY_LIMITS) diff --git a/litellm/llms/openrouter/common_utils.py b/litellm/llms/openrouter/common_utils.py index 64caf83d745..ebdb47b5545 100644 --- a/litellm/llms/openrouter/common_utils.py +++ b/litellm/llms/openrouter/common_utils.py @@ -1,13 +1,17 @@ from collections.abc import Mapping, Sequence -from typing import Final +from types import MappingProxyType +from typing import Annotated, Final import httpx -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, BeforeValidator, ConfigDict import litellm +from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import TokenLimit, get_cached_model_limits from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -16,16 +20,43 @@ _CHAT_CATALOG_ROUTE: Final = "models/user" _CHAT_COMPLETIONS_PATH: Final = "/chat/completions" _BATCH_ONLY_SUFFIX: Final = ":batch" _CATALOG_TIMEOUT: Final = httpx.Timeout(10.0) +_NO_LIMITS: Final[Mapping[str, int]] = MappingProxyType({}) class OpenRouterException(BaseLLMException): pass +def _mapping_or_none(value: object) -> object: + return value if isinstance(value, Mapping) else None + + +class _TopProvider(BaseModel): + model_config = ConfigDict(frozen=True) + + max_completion_tokens: TokenLimit = None + + class _CatalogModel(BaseModel): model_config = ConfigDict(frozen=True) id: str + context_length: TokenLimit = None + top_provider: Annotated[_TopProvider | None, BeforeValidator(_mapping_or_none)] = None + + def token_limits(self) -> Mapping[str, int]: + output_limit: Final = self.top_provider.max_completion_tokens if self.top_provider is not None else None + return MappingProxyType( + { + key: value + for key, value in ( + ("max_tokens", output_limit or self.context_length), + ("max_input_tokens", self.context_length), + ("max_output_tokens", output_limit), + ) + if value is not None + } + ) class _Catalog(BaseModel): @@ -38,6 +69,24 @@ def _chat_catalog_url(api_base: str) -> str: return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE) +def _limits_by_catalog_id(content: bytes) -> Mapping[str, Mapping[str, int]]: + catalog: Final = _Catalog.model_validate_json(content) + return MappingProxyType({model.id: model.token_limits() for model in catalog.data}) + + +async def aget_openrouter_catalog_limits( + *, api_base: str, headers: Mapping[str, str], client: AsyncHTTPHandler, cache: InMemoryCache +) -> Mapping[str, Mapping[str, int]]: + return await get_cached_model_limits( + url=_chat_catalog_url(api_base), + headers=headers, + parse=_limits_by_catalog_id, + client=client, + cache=cache, + timeout=_CATALOG_TIMEOUT, + ) + + class OpenRouterModelInfo(BaseLLMModelInfo): @staticmethod def get_api_key(api_key: str | None = None) -> str | None: @@ -90,3 +139,9 @@ class OpenRouterModelInfo(BaseLLMModelInfo): for model in _Catalog.model_validate_json(response.content).data if not model.id.endswith(_BATCH_ONLY_SUFFIX) ] + + +def openrouter_catalog_limits_for_model(catalog: Mapping[str, Mapping[str, int]], model: str) -> Mapping[str, int]: + catalog_id: Final = OpenRouterModelInfo.get_base_model(model) + candidates: Final = (catalog_id, model, catalog_id.partition(":")[0]) + return next((limits for limits in map(catalog.get, candidates) if limits), _NO_LIMITS) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5e3ed51474c..80f4f3d814e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16155,7 +16155,7 @@ def _enrich_model_info_with_litellm_data( except Exception: litellm_model_info = {} discovered_model_info: Final = ( - llm_router.registry().discovered_model_info(model_info.get("id")) + llm_router.registry().discovered_model_info(model_info.get("id"), model.get("litellm_params", {}).get("model")) if llm_router is not None else MappingProxyType({}) ) diff --git a/litellm/router.py b/litellm/router.py index 20af2e5649e..93c37e7973d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -182,6 +182,11 @@ from litellm.llms.openai_like.model_info import ( MODEL_INFO_REFRESH_SECONDS, get_openai_compatible_model_info, ) +from litellm.llms.openrouter.common_utils import ( + OpenRouterModelInfo, + aget_openrouter_catalog_limits, + openrouter_catalog_limits_for_model, +) from litellm.responses.bridge_check import keep_registered_mode from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_api import ( @@ -12240,53 +12245,121 @@ class Router(DeprecatedRouterMembers): ) ) model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) - if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + if params.get("use_clientside_credentials"): return - if api_base is None or "*" in model or params.get("use_clientside_credentials"): + discovered: Final = await ( + self._adiscover_openrouter_model_info( + raw_deployment, params, api_key=params.api_key or dynamic_api_key, api_base=api_base, client=client + ) + if provider == LlmProviders.OPENROUTER.value + else self._adiscover_openai_compatible_model_info( + raw_deployment, + params, + model=model, + provider=provider, + api_key=params.api_key or dynamic_api_key, + api_base=api_base, + client=client, + ) + ) + model_id: Final = deployment.model_info.id + if ( + discovered is None + or not (discovered.limits or discovered.limits_by_model) + or model_id is None + or self._core_get_model_info(model_id) is not raw_deployment + ): return - api_key: Final = await aresolve_api_key_secret_reference( - params.api_key or dynamic_api_key, + self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1) + self._discovered_model_info_cache.delete_cache(model_id) + self._discovered_model_info_cache.set_cache(model_id, discovered) + self._invalidate_model_group_info_cache() + + async def _adiscover_openai_compatible_model_info( + self, + raw_deployment: Mapping[str, object], + params: LiteLLM_Params, + *, + model: str, + provider: str, + api_key: str | None, + api_base: str | None, + client: AsyncHTTPHandler | None, + ) -> DiscoveredDeploymentModelInfo | None: + if provider not in MODEL_INFO_DISCOVERY_PROVIDERS or api_base is None or "*" in model: + return None + resolved_key: Final = await aresolve_api_key_secret_reference( + api_key, model=model, llm_provider=provider, upstream=MappingProxyType({"api_base": params.api_base}) + ) + limits: Final = await get_openai_compatible_model_info( model=model, - llm_provider=provider, + api_base=api_base, + headers=self._model_info_request_headers(params, resolved_key), + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self._core_cache.in_memory_cache, + ) + return DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits) + + async def _adiscover_openrouter_model_info( + self, + raw_deployment: Mapping[str, object], + params: LiteLLM_Params, + *, + api_key: str | None, + api_base: str | None, + client: AsyncHTTPHandler | None, + ) -> DiscoveredDeploymentModelInfo | None: + resolved_key: Final = await aresolve_api_key_secret_reference( + OpenRouterModelInfo.get_api_key(api_key), + model=params.model, + llm_provider=LlmProviders.OPENROUTER.value, upstream=MappingProxyType({"api_base": params.api_base}), ) + if not resolved_key: + return None + catalog: Final = await aget_openrouter_catalog_limits( + api_base=OpenRouterModelInfo.get_api_base(api_base), + headers=self._model_info_request_headers(params, resolved_key), + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENROUTER), + cache=self._core_cache.in_memory_cache, + ) + if "*" in params.model: + return DiscoveredDeploymentModelInfo( + deployment=raw_deployment, limits=MappingProxyType({}), limits_by_model=catalog + ) + return DiscoveredDeploymentModelInfo( + deployment=raw_deployment, limits=openrouter_catalog_limits_for_model(catalog, params.model) + ) + + @staticmethod + def _model_info_request_headers(params: LiteLLM_Params, api_key: str | None) -> Mapping[str, str]: headers: Final = TypeAdapter(Mapping[str, str]).validate_python( params.get("extra_headers") or params.get("headers") or MappingProxyType({}) ) auth_headers: Final = ( MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) ) - limits: Final = await get_openai_compatible_model_info( - model=model, - api_base=api_base, - headers=MappingProxyType( - { - **auth_headers, - **MappingProxyType({key.lower(): value for key, value in headers.items()}), - } - ), - client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), - cache=self._core_cache.in_memory_cache, + return MappingProxyType( + { + **auth_headers, + **MappingProxyType({key.lower(): value for key, value in headers.items()}), + } ) - model_id: Final = deployment.model_info.id - if not limits or model_id is None or self._core_get_model_info(model_id) is not raw_deployment: - return - self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1) - self._discovered_model_info_cache.delete_cache(model_id) - self._discovered_model_info_cache.set_cache( - model_id, DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits) - ) - self._invalidate_model_group_info_cache() - def _core_get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + def _core_get_discovered_model_info(self, model_id: str | None, model: str | None = None) -> Mapping[str, int]: cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) if ( model_id is not None and isinstance(cached, DiscoveredDeploymentModelInfo) and cached.deployment is self._core_get_model_info(model_id) ): - configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) - return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) + configured: Final = _MODEL_INFO_ADAPTER.validate_python(cached.deployment["model_info"]) + limits: Final = ( + openrouter_catalog_limits_for_model(cached.limits_by_model, model) + if model is not None and cached.limits_by_model + else cached.limits + ) + return MappingProxyType({key: value for key, value in limits.items() if configured.get(key) is None}) return MappingProxyType({}) def _core_get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: @@ -12296,21 +12369,22 @@ class Router(DeprecatedRouterMembers): configured or discovered token limits. Resolved via O(1) index lookup. - Returns None for wildcard-expanded or unknown names, where the listed name is the - real model name and no deployment-specific information exists, and treats a - malformed configured limit as absent rather than failing the listing. + Returns None for unknown names and for wildcard-expanded names whose wildcard + deployment holds no catalog row for them, where the listed name is the real model + name and no deployment-specific information exists, and treats a malformed + configured limit as absent rather than failing the listing. The whole group is read rather than just its first deployment, so a group that mixes models does not advertise a window that depends on config order; the widest one is reported, which is what get_model_group_info already shows the Admin UI. Keys are deduplicated, so the ordinary group of interchangeable deployments of one model still costs the caller a single cost-map lookup. Unlike get_model_group_info, - this never triggers pattern matching or deep copies, so it is safe to call per - listed model on the /v1/models hot path. + this never deep copies a deployment, so it is safe to call per listed model on the + /v1/models hot path. """ deployments: Final = self._core_deployments_named(model_name) if not deployments: - return None + return self._core_get_wildcard_listing_info(model_name) model_infos: Final = tuple( MappingProxyType( @@ -12338,6 +12412,26 @@ class Router(DeprecatedRouterMembers): max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), ) + def _core_get_wildcard_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: + model_infos: Final = tuple(self._wildcard_catalog_model_infos(model_name)) + if not model_infos: + return None + return DeploymentModelListingInfo( + cost_map_keys=(), + max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), + max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + ) + + def _wildcard_catalog_model_infos(self, model_name: str) -> Iterator[Mapping[str, object]]: + for deployment, routed_model in self._core_pattern_router.get_routed_deployments(model_name): + configured = _configured_model_info(deployment) + model_id = configured.get("id") + discovered = self._core_get_discovered_model_info( + model_id if isinstance(model_id, str) else None, routed_model + ) + if discovered: + yield MappingProxyType({**configured, **discovered}) + @staticmethod def _widest_configured_limit(model_infos: Sequence[Mapping[str, object]], field: str) -> int | None: """The largest usable value of ``field`` across a group's configured model_info blocks.""" @@ -12776,7 +12870,7 @@ class Router(DeprecatedRouterMembers): custom_model_info = ( { **copy.deepcopy(litellm.model_cost.get(model_id) or MappingProxyType({})), - **self._core_get_discovered_model_info(model_id), + **self._core_get_discovered_model_info(model_id, model_name), } if model_id in litellm.model_cost else None diff --git a/litellm/router_api.py b/litellm/router_api.py index d62999cf261..4f05bcba619 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -247,9 +247,12 @@ class RouterRegistry: id=id, # pyright: ignore[reportArgumentType] # see above ) - def discovered_model_info(self, id: str | None) -> Mapping[str, int]: - """Limits discovered from the provider for a deployment (was ``get_discovered_model_info``).""" - return self._router._core_get_discovered_model_info(id) + def discovered_model_info(self, id: str | None, model: str | None = None) -> Mapping[str, int]: + """Limits discovered from the provider for a deployment (was ``get_discovered_model_info``). + + ``model`` is the concrete model a wildcard deployment serves, which selects its row in the catalog. + """ + return self._router._core_get_discovered_model_info(id, model) def zero_cost(self) -> Mapping[str, bool] | None: """The per-deployment zero-cost lookup the auth checks read (was ``_zero_cost_cache``).""" diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 046fb8c452a..7d9860d8623 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -420,9 +420,9 @@ class DeprecatedRouterMembers: id=id, # pyright: ignore[reportArgumentType] # see above ) - def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + def get_discovered_model_info(self, model_id: str | None, model: str | None = None) -> Mapping[str, int]: _note("get_discovered_model_info", "registry().discovered_model_info(id)") - return _as_router(self)._core_get_discovered_model_info(model_id) + return _as_router(self)._core_get_discovered_model_info(model_id, model) # -- registry: many deployments ------------------------------------------------------------- def get_model_list( diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 2b8d1e9c11b..cf727e35321 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching import copy import itertools import re +from collections.abc import Mapping from re import Match from typing import Final @@ -260,8 +261,10 @@ class PatternMatchRouter: return pattern_match return [] - def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]: - """The models get_deployments_by_pattern would route ``model`` to, without copying the deployments.""" + def get_routed_deployments( + self, model: str, custom_llm_provider: str | None = None + ) -> tuple[tuple[Mapping[str, object], str], ...]: + """Each deployment get_deployments_by_pattern would route ``model`` to, uncopied, with its concrete model.""" provider: Final = custom_llm_provider or inferred_provider(model) requests_by_pattern: Final = itertools.product((model, f"{provider}/{model}"), self.patterns) matched: Final = next( @@ -273,17 +276,25 @@ class PatternMatchRouter: None, ) if matched is None: - return frozenset() + return () pattern_match, pattern = matched - return frozenset( - PatternMatchRouter.set_deployment_model_name( - matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"] + return tuple( + ( + deployment, + PatternMatchRouter.set_deployment_model_name( + matched_pattern=pattern_match, + litellm_deployment_litellm_model=deployment["litellm_params"]["model"], + ), ) for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts pattern, () ) ) + def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]: + """The models get_deployments_by_pattern would route ``model`` to, without copying the deployments.""" + return frozenset(routed_model for _, routed_model in self.get_routed_deployments(model, custom_llm_provider)) + # Example usage: # router = PatternRouter() diff --git a/litellm/types/router.py b/litellm/types/router.py index f927ae0475b..aa2f8c2f353 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -917,6 +917,7 @@ class Deployment(LiteLLMBaseModel): class DiscoveredDeploymentModelInfo: deployment: Mapping[str, object] limits: Mapping[str, int] + limits_by_model: Mapping[str, Mapping[str, int]] = field(default_factory=lambda: MappingProxyType({})) @dataclass(frozen=True, slots=True) diff --git a/tests/integration/providers/_cache_control_marks_support.py b/tests/integration/providers/_cache_control_marks_support.py index 2f6071f9e45..bca8a7f9e89 100644 --- a/tests/integration/providers/_cache_control_marks_support.py +++ b/tests/integration/providers/_cache_control_marks_support.py @@ -464,6 +464,7 @@ def owned_config( *, litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}), router_settings: Mapping[str, JsonValue] = MappingProxyType({}), + general_settings: Mapping[str, JsonValue] = MappingProxyType({}), ) -> Path: config: Final = _JSON_OBJECT.validate_python( yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) @@ -473,6 +474,7 @@ def owned_config( "model_list": list(model_list), "litellm_settings": {**object_value(config["litellm_settings"]), **litellm_settings}, "router_settings": {**object_value(config["router_settings"]), "num_retries": 0, **router_settings}, + "general_settings": {**object_value(config["general_settings"]), **general_settings}, } path: Final = directory / f"cache-control-marks-{uuid.uuid4().hex}.yaml" path.write_text(yaml.safe_dump(merged)) diff --git a/tests/integration/routing/test_openrouter_catalog_limits_wire.py b/tests/integration/routing/test_openrouter_catalog_limits_wire.py new file mode 100644 index 00000000000..6404591be7c --- /dev/null +++ b/tests/integration/routing/test_openrouter_catalog_limits_wire.py @@ -0,0 +1,109 @@ +import json +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import ( + Gateway, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._cache_control_marks_support import owned_config +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_API_KEY: Final = "integration-openrouter-key" +_CATALOG_ROUTE: Final = "/api/v1/models/user" +_CATALOG: Final = { + "data": [ + {"id": "vendor/capped", "context_length": 4096, "top_provider": {"max_completion_tokens": 1024}}, + {"id": "vendor/uncapped", "context_length": 8192, "top_provider": {"max_completion_tokens": None}}, + ] +} +_NAMED_MODEL: Final = "gpt-mini-named" +_EXPECTED_LIMITS: Final = { + _NAMED_MODEL: (4096, 1024), + "openrouter/vendor/capped": (4096, 1024), + "openrouter/vendor/uncapped": (8192, None), +} + + +@dataclass(frozen=True, slots=True) +class _CatalogProxy: + gateway: Gateway + wire: Wire + + +def _catalog_reply(_request: Request) -> Reply: + return Reply(body=json.dumps(_CATALOG).encode()) + + +def _deployment(model_name: str, model: str, wire: Wire) -> JsonValue: + return { + "model_name": model_name, + "litellm_params": {"model": model, "api_key": _API_KEY, "api_base": f"{wire.url}/api/v1"}, + } + + +@pytest.fixture(scope="module") +def catalog_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_CatalogProxy]: + directory: Final[Path] = tmp_path_factory.mktemp("openrouter-catalog-limits") + with gateway_from_environment() as gateway, wire_server(_catalog_reply) as wire: + config: Final = owned_config( + directory, + [ + _deployment(_NAMED_MODEL, "openrouter/vendor/capped", wire), + _deployment("openrouter/*", "openrouter/*", wire), + ], + litellm_settings={"check_provider_endpoint": True}, + general_settings={"disable_model_info_refresh": False}, + ) + with owned_proxy_process(gateway, directory, {}, config=config) as owned: + yield _CatalogProxy(gateway=owned.gateway, wire=wire) + + +def _listed_limits( + gateway: Gateway, path: str, name_key: str, row_limits: str | None +) -> Mapping[str, tuple[JsonValue, JsonValue]]: + rows: Final = tuple(object_value(row) for row in list_value(gateway.get(path)["data"])) + named_rows: Final = {string_value(row[name_key]): row for row in rows} + limit_rows: Final = { + name: object_value(row[row_limits]) if row_limits is not None else row for name, row in named_rows.items() + } + return { + name: (limit_rows[name].get("max_input_tokens"), limit_rows[name].get("max_output_tokens")) + for name in _EXPECTED_LIMITS + if name in limit_rows + } + + +@pytest.mark.parametrize( + ("path", "name_key", "row_limits"), + ( + ("/v1/model/info", "model_name", "model_info"), + ("/model_group/info", "model_group", None), + ("/v1/models", "id", None), + ), +) +def test_openrouter_catalog_limits_reach_the_model_info_routes_with_no_cost_map_row( + catalog_proxy: _CatalogProxy, path: str, name_key: str, row_limits: str | None +) -> None: + limits: Final = eventually( + lambda: _listed_limits(catalog_proxy.gateway, path, name_key, row_limits), + lambda observed: observed == _EXPECTED_LIMITS, + seconds=30, + ) + + assert limits == _EXPECTED_LIMITS + catalog_calls: Final = catalog_proxy.wire.drain() + assert {(call.method, call.target, call.headers["authorization"]) for call in catalog_calls} <= { + ("GET", _CATALOG_ROUTE, f"Bearer {_API_KEY}") + } diff --git a/tests/unit/llms/openai_like/test_model_info.py b/tests/unit/llms/openai_like/test_model_info.py index 15a7d9e7fc6..0a01580e888 100644 --- a/tests/unit/llms/openai_like/test_model_info.py +++ b/tests/unit/llms/openai_like/test_model_info.py @@ -124,3 +124,19 @@ async def test_unavailable_metadata_is_best_effort_and_negative_cached( == {} ) assert responder.call_count == 1 + + +async def test_first_card_wins_when_an_endpoint_repeats_a_model_id() -> None: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, json={"data": [{"id": "model", "max_model_len": 4096}, {"id": "model", "max_model_len": 8192}]} + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + limits: Final = await get_openai_compatible_model_info( + model="model", api_base="https://backend.test", headers={}, client=handler, cache=InMemoryCache() + ) + assert limits == {"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096} diff --git a/tests/unit/llms/openrouter/test_openrouter_common_utils.py b/tests/unit/llms/openrouter/test_openrouter_common_utils.py index 087c05561b5..c330b971b38 100644 --- a/tests/unit/llms/openrouter/test_openrouter_common_utils.py +++ b/tests/unit/llms/openrouter/test_openrouter_common_utils.py @@ -1,11 +1,19 @@ +from collections.abc import Mapping from typing import Final +from unittest.mock import Mock import httpx import pytest import respx import litellm -from litellm.llms.openrouter.common_utils import OpenRouterModelInfo +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openrouter.common_utils import ( + OpenRouterModelInfo, + aget_openrouter_catalog_limits, + openrouter_catalog_limits_for_model, +) DEFAULT_BASE: Final = "https://openrouter.ai/api/v1" KEY_SOURCES: Final = ("api_key", "litellm.api_key", "litellm.openrouter_key", "OPENROUTER_API_KEY", "OR_API_KEY") @@ -157,3 +165,106 @@ def test_validate_environment_without_any_key_sends_no_authorization(): ) def test_get_base_model_drops_only_the_openrouter_prefix(model: str, base_model: str): assert OpenRouterModelInfo.get_base_model(model) == base_model + + +@respx.mock +def test_get_models_keeps_listing_when_a_catalog_row_carries_unusable_limits(): + respx.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=httpx.Response( + 200, + json={ + "data": [ + {"id": "vendor/junk-limits", "context_length": "many", "top_provider": "none"}, + {"id": "vendor/junk-cap", "top_provider": {"max_completion_tokens": "many"}}, + {"id": "vendor/no-limits"}, + ] + }, + ) + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ + "openrouter/vendor/junk-limits", + "openrouter/vendor/junk-cap", + "openrouter/vendor/no-limits", + ] + + +@pytest.mark.parametrize( + ("row", "expected"), + ( + ( + {"context_length": 4096, "top_provider": {"max_completion_tokens": 1024}}, + {"max_tokens": 1024, "max_input_tokens": 4096, "max_output_tokens": 1024}, + ), + ( + {"context_length": 4096, "top_provider": {"max_completion_tokens": None}}, + {"max_tokens": 4096, "max_input_tokens": 4096}, + ), + ({"context_length": 4096}, {"max_tokens": 4096, "max_input_tokens": 4096}), + ( + {"context_length": 4096, "top_provider": {"context_length": 8192, "max_completion_tokens": 1024}}, + {"max_tokens": 1024, "max_input_tokens": 4096, "max_output_tokens": 1024}, + ), + ({"top_provider": {"max_completion_tokens": 1024}}, {"max_tokens": 1024, "max_output_tokens": 1024}), + ({"context_length": 0, "top_provider": {"max_completion_tokens": True}}, {}), + ({"context_length": "many", "top_provider": "none"}, {}), + ), +) +async def test_catalog_row_becomes_the_published_limits_and_never_invents_an_output_cap( + row: Mapping[str, object], expected: Mapping[str, int] +) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == f"{DEFAULT_BASE}/models/user" + assert request.headers["authorization"] == "Bearer sk-or-test" + return httpx.Response(200, json={"data": [{"id": "vendor/model", **row}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + catalog: Final = await aget_openrouter_catalog_limits( + api_base=DEFAULT_BASE, + headers={"Authorization": "Bearer sk-or-test"}, + client=handler, + cache=InMemoryCache(), + ) + assert catalog == {"vendor/model": expected} + + +async def test_a_failed_catalog_fetch_is_cached_so_the_next_refresh_does_not_call_again() -> None: + responder: Final = Mock(return_value=httpx.Response(503)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + cache: Final = InMemoryCache() + for _ in range(2): + assert ( + await aget_openrouter_catalog_limits(api_base=DEFAULT_BASE, headers={}, client=handler, cache=cache) + == {} + ) + assert responder.call_count == 1 + + +@pytest.mark.parametrize( + ("model", "expected"), + ( + ("openrouter/vendor/model", {"max_input_tokens": 1}), + ("vendor/model", {"max_input_tokens": 1}), + ("openrouter/vendor/model:nitro", {"max_input_tokens": 1}), + ("openrouter/vendor/model:free", {"max_input_tokens": 2}), + ("openrouter/auto", {"max_input_tokens": 3}), + ("openrouter/vendor/unlisted", {}), + ("openrouter/vendor/unlisted:nitro", {}), + ), +) +def test_catalog_lookup_finds_a_model_by_its_id_then_by_the_id_without_its_variant( + model: str, expected: Mapping[str, int] +) -> None: + catalog: Final = { + "vendor/model": {"max_input_tokens": 1}, + "vendor/model:free": {"max_input_tokens": 2}, + "openrouter/auto": {"max_input_tokens": 3}, + } + + assert openrouter_catalog_limits_for_model(catalog, model) == expected diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index bffa4612964..902e2361558 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import respx from fastapi.testclient import TestClient import litellm @@ -165,6 +166,93 @@ async def test_upstream_limits_reach_model_info_routes( _invalidate_model_cost_lowercase_map() +async def test_openrouter_catalog_limits_reach_every_model_info_route( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "named", + "litellm_params": {"model": "openrouter/vendor/capped", "api_key": "openrouter-key"}, + "model_info": {"id": "named-deployment"}, + }, + { + "model_name": "overridden", + "litellm_params": {"model": "openrouter/vendor/capped", "api_key": "openrouter-key"}, + "model_info": {"id": "overridden-deployment", "max_output_tokens": 512}, + }, + { + "model_name": "openrouter/*", + "litellm_params": {"model": "openrouter/*", "api_key": "openrouter-key"}, + "model_info": {"id": "wildcard-deployment"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + catalog: Final = { + "data": [ + {"id": "vendor/capped", "context_length": 4096, "top_provider": {"max_completion_tokens": 1024}}, + {"id": "vendor/uncapped", "context_length": 8192, "top_provider": {"max_completion_tokens": None}}, + ] + } + expected: Final = { + "named": (4096, 1024), + "overridden": (4096, 512), + "openrouter/vendor/capped": (4096, 1024), + "openrouter/vendor/uncapped": (8192, None), + } + + def limits_by_name(rows: list[dict[str, object]], name_key: str) -> dict[object, tuple[object, object]]: + return { + row[name_key]: (row.get("max_input_tokens"), row.get("max_output_tokens")) + for row in rows + if row[name_key] in expected + } + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json=catalog)) + ) as upstream: + handler.client = upstream + litellm.in_memory_llm_clients_cache.set_cache("async_httpx_clientopenrouter", handler) + await proxy_server.ProxyStartupEvent.refresh_model_info() + with respx.mock(assert_all_called=False) as sync_upstream, auth_as(): + sync_upstream.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json=catalog) + ) + for path in ("/v1/model/info", "/model/info"): + info_response: Final = client.get(path) + assert info_response.status_code == 200, info_response.text + assert ( + limits_by_name( + [ + {"model_name": row["model_name"], **row["model_info"]} + for row in info_response.json()["data"] + ], + "model_name", + ) + == expected + ) + group_response: Final = client.get("/model_group/info") + assert group_response.status_code == 200, group_response.text + assert limits_by_name(group_response.json()["data"], "model_group") == expected + models_response: Final = client.get("/v1/models") + assert models_response.status_code == 200, models_response.text + assert limits_by_name(models_response.json()["data"], "id") == expected + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /v2/model/info # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index 41acd487b67..d286f8e3371 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -142,7 +142,7 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: registry.deployment.side_effect = lambda id: router.get_deployment(model_id=id) registry.deployment_dict.side_effect = lambda id: router.get_model_info(id=id) registry.has_deployment.side_effect = lambda id: router.has_model_id(id) - registry.discovered_model_info.side_effect = lambda id: router.get_discovered_model_info(id) + registry.discovered_model_info.side_effect = lambda id, model=None: router.get_discovered_model_info(id, model) def _credentials(id, *, form="params", **team): # Old call sites passed team_id (even None) exactly when the new ones do, so pass it on as given. diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 2af7db930c6..55ce920422d 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -585,7 +585,7 @@ FORWARDS = [ "_core_get_router_model_info", ((), {"deployment": None, "received_model_name": "gpt", "id": "dep-a"}), ), - (lambda r: r.registry().discovered_model_info("dep-a"), "_core_get_discovered_model_info", (("dep-a",), {})), + (lambda r: r.registry().discovered_model_info("dep-a"), "_core_get_discovered_model_info", (("dep-a", None), {})), ( lambda r: r.registry().deployments("gpt", source="routing_groups"), "_core_get_model_list_from_routing_groups", diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 91f0b2339c7..d24530813a9 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -19,6 +19,7 @@ import re import sys import threading from pathlib import Path +from collections.abc import Callable, Iterator from types import MappingProxyType from typing import Final, cast from unittest.mock import Mock, patch @@ -37,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.types.router import ( Deployment, + DeploymentModelListingInfo, DeploymentTypedDict, LiteLLM_Params, LiteLLMParamsTypedDict, @@ -456,6 +458,351 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa _invalidate_model_cost_lowercase_map() +@pytest.fixture +def openrouter_settings_cleared(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for env_var in ("OPENROUTER_API_BASE", "OPENROUTER_API_KEY", "OR_API_KEY"): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "openrouter_key", None) + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + yield + _invalidate_model_cost_lowercase_map() + + +def _openrouter_catalog_response(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/models/user") + return httpx.Response( + 200, + json={ + "data": [ + {"id": "vendor/capped", "context_length": 4096, "top_provider": {"max_completion_tokens": 1024}}, + {"id": "vendor/uncapped", "context_length": 8192, "top_provider": {"max_completion_tokens": None}}, + ] + }, + ) + + +def _openrouter_deployment( + model_name: str, model: str, deployment_id: str, **model_info: int | None +) -> DeploymentTypedDict: + return { + "model_name": model_name, + "litellm_params": {"model": model, "api_key": "openrouter-key"}, + "model_info": {"id": deployment_id, **model_info}, + } + + +async def _refresh_against(router: Router, respond: Callable[[httpx.Request], httpx.Response]) -> None: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + + +def _group_limits(router: Router, model_group: str) -> tuple[int | None, int | None]: + group: Final = router.get_model_group_info(model_group) + assert group is not None + return group.max_input_tokens, group.max_output_tokens + + +PUBLISHED_OPENROUTER_LIMITS: Final = ( + ("vendor/capped", (4096, 1024)), + ("vendor/capped:nitro", (4096, 1024)), + ("vendor/uncapped", (8192, None)), + ("vendor/unlisted", (None, None)), +) + + +@pytest.mark.parametrize(("catalog_id", "expected"), PUBLISHED_OPENROUTER_LIMITS) +async def test_openrouter_named_deployment_shows_the_limits_the_catalog_publishes( + catalog_id: str, expected: tuple[int | None, int | None], openrouter_settings_cleared: None +) -> None: + router: Final = Router(model_list=[_openrouter_deployment("named", f"openrouter/{catalog_id}", "named-id")]) + + await _refresh_against(router, _openrouter_catalog_response) + + assert _group_limits(router, "named") == expected + + +@pytest.mark.parametrize(("catalog_id", "expected"), PUBLISHED_OPENROUTER_LIMITS) +async def test_openrouter_wildcard_deployment_shows_the_limits_of_each_concrete_model( + catalog_id: str, expected: tuple[int | None, int | None], openrouter_settings_cleared: None +) -> None: + router: Final = Router(model_list=[_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id")]) + + await _refresh_against(router, _openrouter_catalog_response) + + assert _group_limits(router, f"openrouter/{catalog_id}") == expected + + +async def test_discovered_model_info_without_a_model_keeps_answering_for_the_deployment_itself( + openrouter_settings_cleared: None, +) -> None: + router: Final = Router( + model_list=[ + _openrouter_deployment("named", "openrouter/vendor/capped", "named-id"), + _openrouter_deployment("openrouter/*", "openrouter/*", "wild-id"), + ] + ) + + await _refresh_against(router, _openrouter_catalog_response) + + own_limits: Final = {"max_tokens": 1024, "max_input_tokens": 4096, "max_output_tokens": 1024} + registry: Final = router.registry() + assert registry.discovered_model_info("named-id") == own_limits + assert registry.discovered_model_info("named-id", "openrouter/vendor/uncapped") == own_limits + assert registry.discovered_model_info("wild-id") == {} + assert registry.discovered_model_info("wild-id", "openrouter/vendor/capped") == own_limits + + +async def test_operator_model_info_beats_the_openrouter_catalog(openrouter_settings_cleared: None) -> None: + router: Final = Router( + model_list=[ + _openrouter_deployment("named", "openrouter/vendor/capped", "named-id", max_input_tokens=100), + _openrouter_deployment("openrouter/*", "openrouter/*", "wild-id", max_output_tokens=50), + ] + ) + + await _refresh_against(router, _openrouter_catalog_response) + + assert _group_limits(router, "named") == (100, 1024) + assert _group_limits(router, "openrouter/vendor/capped") == (4096, 50) + + +@pytest.mark.parametrize( + ("deployment", "model_group"), + ( + (("named", "openrouter/vendor/capped", "named-id"), "named"), + (("openrouter/*", "openrouter/*", "wild-id"), "openrouter/vendor/capped"), + ), +) +async def test_openrouter_catalog_beats_a_cost_map_row_for_a_listed_model( + deployment: tuple[str, str, str], model_group: str, openrouter_settings_cleared: None +) -> None: + litellm.model_cost["openrouter/vendor/capped"] = { + "litellm_provider": "openrouter", + "mode": "chat", + "max_input_tokens": 1, + "max_output_tokens": 2, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + _invalidate_model_cost_lowercase_map() + router: Final = Router(model_list=[_openrouter_deployment(*deployment)]) + + await _refresh_against(router, _openrouter_catalog_response) + + assert _group_limits(router, model_group) == (4096, 1024) + + +@pytest.mark.parametrize( + ("deployment", "model_group"), + ( + (("named", "openrouter/vendor/unlisted", "named-id"), "named"), + (("openrouter/*", "openrouter/*", "wild-id"), "openrouter/vendor/unlisted"), + ), +) +async def test_openrouter_cost_map_row_stays_for_a_model_the_catalog_does_not_list( + deployment: tuple[str, str, str], model_group: str, openrouter_settings_cleared: None +) -> None: + litellm.model_cost["openrouter/vendor/unlisted"] = { + "litellm_provider": "openrouter", + "mode": "chat", + "max_input_tokens": 1, + "max_output_tokens": 2, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + _invalidate_model_cost_lowercase_map() + router: Final = Router(model_list=[_openrouter_deployment(*deployment)]) + + await _refresh_against(router, _openrouter_catalog_response) + + assert _group_limits(router, model_group) == (1, 2) + + +async def test_openrouter_group_takes_the_widest_published_limit_with_no_cost_data( + openrouter_settings_cleared: None, +) -> None: + router: Final = Router( + model_list=[ + _openrouter_deployment("group", "openrouter/vendor/capped", "capped-id"), + _openrouter_deployment("group", "openrouter/vendor/uncapped", "uncapped-id"), + ] + ) + + await _refresh_against(router, _openrouter_catalog_response) + + assert _group_limits(router, "group") == (8192, 1024) + + +async def test_openrouter_deployments_on_one_key_share_one_catalog_fetch(openrouter_settings_cleared: None) -> None: + router: Final = Router( + model_list=[ + _openrouter_deployment("first", "openrouter/vendor/capped", "first-id"), + _openrouter_deployment("second", "openrouter/vendor/uncapped", "second-id"), + _openrouter_deployment("openrouter/*", "openrouter/*", "wild-id"), + ] + ) + responder: Final = Mock(side_effect=_openrouter_catalog_response) + + await _refresh_against(router, responder) + + assert responder.call_count == 1 + assert _group_limits(router, "first") == (4096, 1024) + assert _group_limits(router, "second") == (8192, None) + assert _group_limits(router, "openrouter/vendor/capped") == (4096, 1024) + + +@pytest.mark.parametrize( + ("deployment", "model_group"), + ( + (("named", "openrouter/vendor/capped", "named-id"), "named"), + (("openrouter/*", "openrouter/*", "wild-id"), "openrouter/vendor/capped"), + ), +) +async def test_an_openrouter_catalog_outage_keeps_the_last_published_limits( + deployment: tuple[str, str, str], model_group: str, openrouter_settings_cleared: None +) -> None: + router: Final = Router(model_list=[_openrouter_deployment(*deployment)]) + + await _refresh_against(router, _openrouter_catalog_response) + router.cache.in_memory_cache.flush_cache() + await _refresh_against(router, lambda request: httpx.Response(503)) + + assert _group_limits(router, model_group) == (4096, 1024) + + +async def test_openrouter_discovery_without_a_key_makes_no_call(openrouter_settings_cleared: None) -> None: + router: Final = Router( + model_list=[ + {"model_name": "named", "litellm_params": {"model": "openrouter/vendor/capped"}}, + {"model_name": "openrouter/*", "litellm_params": {"model": "openrouter/*"}}, + ] + ) + responder: Final = Mock(side_effect=_openrouter_catalog_response) + + await _refresh_against(router, responder) + + assert responder.call_count == 0 + assert _group_limits(router, "named") == (None, None) + + +async def test_openrouter_discovery_skips_a_caller_credentials_deployment( + openrouter_settings_cleared: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "environment-key") + router: Final = Router( + model_list=[ + { + "model_name": "byok", + "litellm_params": {"model": "openrouter/vendor/capped", "use_clientside_credentials": True}, + } + ] + ) + responder: Final = Mock(side_effect=_openrouter_catalog_response) + + await _refresh_against(router, responder) + + assert responder.call_count == 0 + assert _group_limits(router, "byok") == (None, None) + + +async def test_openrouter_discovery_sends_the_secret_behind_an_oidc_key_to_the_configured_base( + openrouter_settings_cleared: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("openrouter-secret") + router: Final = Router( + model_list=[ + { + "model_name": "named", + "litellm_params": { + "model": "openrouter/vendor/capped", + "api_base": "https://openrouter.test/tenant/v1", + "api_key": f"oidc/file/{token}", + }, + } + ] + ) + requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return _openrouter_catalog_response(request) + + await _refresh_against(router, respond) + + assert [str(request.url) for request in requests] == ["https://openrouter.test/tenant/v1/models/user"] + assert requests[0].headers["authorization"] == "Bearer openrouter-secret" + assert _group_limits(router, "named") == (4096, 1024) + + +async def test_openrouter_discovery_uses_the_key_from_the_environment( + openrouter_settings_cleared: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "environment-key") + router: Final = Router( + model_list=[{"model_name": "named", "litellm_params": {"model": "openrouter/vendor/capped"}}] + ) + authorizations: Final[list[str | None]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + authorizations.append(request.headers.get("Authorization")) + return _openrouter_catalog_response(request) + + await _refresh_against(router, respond) + + assert authorizations == ["Bearer environment-key"] + + +async def test_a_wildcard_on_another_provider_is_still_not_discovered(openrouter_settings_cleared: None) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "hosted/*", + "litellm_params": { + "model": "hosted_vllm/*", + "api_base": "https://backend.test/v1", + "api_key": "local-key", + }, + } + ] + ) + responder: Final = Mock(return_value=httpx.Response(200, json={"data": []})) + + await _refresh_against(router, responder) + + assert responder.call_count == 0 + assert router.registry().group("hosted/model", field="listing_info") is None + + +async def test_openrouter_wildcard_listing_info_shows_each_concrete_models_limits( + openrouter_settings_cleared: None, +) -> None: + router: Final = Router(model_list=[_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id")]) + + await _refresh_against(router, _openrouter_catalog_response) + + listing_info: Final = router.registry().group("openrouter/vendor/capped", field="listing_info") + assert listing_info == DeploymentModelListingInfo(cost_map_keys=(), max_input_tokens=4096, max_output_tokens=1024) + assert router.registry().group("openrouter/vendor/unlisted", field="listing_info") is None + + +async def test_openrouter_wildcard_listing_info_keeps_the_operator_limit(openrouter_settings_cleared: None) -> None: + router: Final = Router( + model_list=[_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id", max_output_tokens=50)] + ) + + await _refresh_against(router, _openrouter_catalog_response) + + listing_info: Final = router.registry().group("openrouter/vendor/capped", field="listing_info") + assert listing_info == DeploymentModelListingInfo(cost_map_keys=(), max_input_tokens=4096, max_output_tokens=50) + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no