scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

fix(openrouter): show published limits on the proxy model-info routes

Browse source 
An OpenRouter model with no cost-map row showed a null context window and
output cap on /v1/model/info and /model_group/info, so lite pi fell back to
128k. The model-info refresh now reads OpenRouter's /models/user catalog, keeps
one shared catalog per wildcard deployment, and looks up each concrete model on
read, with no new call on the request path. Operator model_info still wins, and
catalog values win over a cost-map row. Wildcard rows on /v1/models show the
same limits, since the lite OpenCode and Codex model sync reads them there
This commit is contained in:
mateo-berri 2026-10-30 04:31:06 -07:00
parent 69ff47d335
commit 9176e4df18
16 changed files with 949 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

90
litellm/llms/openai_like/model_info.py
View file

@ -1,6 +1,7 @@
import hashlib
import json
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Annotated, Final, TypeAlias
@ -17,23 +18,24 @@ MODEL_INFO_REFRESH_SECONDS: Final = 300
MODEL_INFO_REFRESH_CONCURRENCY: Final = 8
MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset({"hosted_vllm", "openai", "text-completion-openai", "openai_like"})
_EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({})
_DISCOVERY_TIMEOUT: Final = httpx.Timeout(5.0)
def _positive_limit(value: object) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
_TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)]
TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)]
class _ModelCard(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
id: str
max_model_len: _TokenLimit = None
context_length: _TokenLimit = None
max_input_tokens: _TokenLimit = None
max_output_tokens: _TokenLimit = None
max_model_len: TokenLimit = None
context_length: TokenLimit = None
max_input_tokens: TokenLimit = None
max_output_tokens: TokenLimit = None
def token_limits(self) -> Mapping[str, int]:
context: Final = self.max_model_len or self.context_length
@ -58,6 +60,51 @@ class _ModelList(LiteLLMBaseModel):
data: tuple[_ModelCard, ...] = ()
@dataclass(frozen=True, slots=True)
class _CachedModelLimits:
by_model: Mapping[str, Mapping[str, int]]
def _limits_of_first_card_per_id(content: bytes) -> Mapping[str, Mapping[str, int]]:
cards: Final = _ModelList.model_validate_json(content).data
return MappingProxyType({card.id: card.token_limits() for card in reversed(cards)})
async def get_cached_model_limits(
*,
url: str,
headers: Mapping[str, str],
parse: Callable[[bytes], Mapping[str, Mapping[str, int]]],
client: AsyncHTTPHandler,
cache: InMemoryCache,
timeout: httpx.Timeout = _DISCOVERY_TIMEOUT,
) -> Mapping[str, Mapping[str, int]]:
cache_key: Final = (
"upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest()
)
cached: Final[object] = cache.get_cache(cache_key)
if isinstance(cached, _CachedModelLimits):
return cached.by_model
try:
response: Final = await client.get(
url=url,
headers=dict(headers),
timeout=timeout,
follow_redirects=False,
max_response_bytes=2 * 1024 * 1024,
)
response.raise_for_status()
limits: Final = parse(response.content)
except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh
verbose_logger.debug("Could not discover upstream model token limits")
cache.set_cache(cache_key, _CachedModelLimits(MappingProxyType({})), ttl=60)
return MappingProxyType({})
cache.set_cache(cache_key, _CachedModelLimits(limits), ttl=MODEL_INFO_REFRESH_SECONDS)
return limits
async def get_openai_compatible_model_info(
*,
model: str,
@ -66,28 +113,11 @@ async def get_openai_compatible_model_info(
client: AsyncHTTPHandler,
cache: InMemoryCache,
) -> Mapping[str, int]:
url: Final = add_path_to_api_base(api_base, "/v1/models")
cache_key: Final = (
"upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest()
limits: Final = await get_cached_model_limits(
url=add_path_to_api_base(api_base, "/v1/models"),
headers=headers,
parse=_limits_of_first_card_per_id,
client=client,
cache=cache,
)
cached: Final[object] = cache.get_cache(cache_key)
if isinstance(cached, _ModelList):
return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS)
try:
response: Final = await client.get(
url=url,
headers=dict(headers),
timeout=httpx.Timeout(5.0),
follow_redirects=False,
max_response_bytes=2 * 1024 * 1024,
)
response.raise_for_status()
models: Final = _ModelList.model_validate_json(response.content)
except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh
verbose_logger.debug("Could not discover upstream model token limits")
cache.set_cache(cache_key, _ModelList(), ttl=60)
return _EMPTY_LIMITS
cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS)
return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS)
return limits.get(model, _EMPTY_LIMITS)

59
litellm/llms/openrouter/common_utils.py
View file

@ -1,13 +1,17 @@
from collections.abc import Mapping, Sequence
from typing import Final
from types import MappingProxyType
from typing import Annotated, Final
import httpx
from pydantic import BaseModel, ConfigDict
from pydantic import BaseModel, BeforeValidator, ConfigDict
import litellm
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.openai_like.model_info import TokenLimit, get_cached_model_limits
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -16,16 +20,43 @@ _CHAT_CATALOG_ROUTE: Final = "models/user"
_CHAT_COMPLETIONS_PATH: Final = "/chat/completions"
_BATCH_ONLY_SUFFIX: Final = ":batch"
_CATALOG_TIMEOUT: Final = httpx.Timeout(10.0)
_NO_LIMITS: Final[Mapping[str, int]] = MappingProxyType({})
class OpenRouterException(BaseLLMException):
pass
def _mapping_or_none(value: object) -> object:
return value if isinstance(value, Mapping) else None
class _TopProvider(BaseModel):
model_config = ConfigDict(frozen=True)
max_completion_tokens: TokenLimit = None
class _CatalogModel(BaseModel):
model_config = ConfigDict(frozen=True)
id: str
context_length: TokenLimit = None
top_provider: Annotated[_TopProvider | None, BeforeValidator(_mapping_or_none)] = None
def token_limits(self) -> Mapping[str, int]:
output_limit: Final = self.top_provider.max_completion_tokens if self.top_provider is not None else None
return MappingProxyType(
{
key: value
for key, value in (
("max_tokens", output_limit or self.context_length),
("max_input_tokens", self.context_length),
("max_output_tokens", output_limit),
)
if value is not None
}
)
class _Catalog(BaseModel):
@ -38,6 +69,24 @@ def _chat_catalog_url(api_base: str) -> str:
return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE)
def _limits_by_catalog_id(content: bytes) -> Mapping[str, Mapping[str, int]]:
catalog: Final = _Catalog.model_validate_json(content)
return MappingProxyType({model.id: model.token_limits() for model in catalog.data})
async def aget_openrouter_catalog_limits(
*, api_base: str, headers: Mapping[str, str], client: AsyncHTTPHandler, cache: InMemoryCache
) -> Mapping[str, Mapping[str, int]]:
return await get_cached_model_limits(
url=_chat_catalog_url(api_base),
headers=headers,
parse=_limits_by_catalog_id,
client=client,
cache=cache,
timeout=_CATALOG_TIMEOUT,
)
class OpenRouterModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
@ -90,3 +139,9 @@ class OpenRouterModelInfo(BaseLLMModelInfo):
for model in _Catalog.model_validate_json(response.content).data
if not model.id.endswith(_BATCH_ONLY_SUFFIX)
]
def openrouter_catalog_limits_for_model(catalog: Mapping[str, Mapping[str, int]], model: str) -> Mapping[str, int]:
catalog_id: Final = OpenRouterModelInfo.get_base_model(model)
candidates: Final = (catalog_id, model, catalog_id.partition(":")[0])
return next((limits for limits in map(catalog.get, candidates) if limits), _NO_LIMITS)

2
litellm/proxy/proxy_server.py
View file

@ -16155,7 +16155,7 @@ def _enrich_model_info_with_litellm_data(
except Exception:
litellm_model_info = {}
discovered_model_info: Final = (
llm_router.registry().discovered_model_info(model_info.get("id"))
llm_router.registry().discovered_model_info(model_info.get("id"), model.get("litellm_params", {}).get("model"))
if llm_router is not None
else MappingProxyType({})
)

164
litellm/router.py
View file

@ -182,6 +182,11 @@ from litellm.llms.openai_like.model_info import (
MODEL_INFO_REFRESH_SECONDS,
get_openai_compatible_model_info,
)
from litellm.llms.openrouter.common_utils import (
OpenRouterModelInfo,
aget_openrouter_catalog_limits,
openrouter_catalog_limits_for_model,
)
from litellm.responses.bridge_check import keep_registered_mode
from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults
from litellm.router_api import (
@ -12240,53 +12245,121 @@ class Router(DeprecatedRouterMembers):
)
)
model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params)
if provider not in MODEL_INFO_DISCOVERY_PROVIDERS:
if params.get("use_clientside_credentials"):
return
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
discovered: Final = await (
self._adiscover_openrouter_model_info(
raw_deployment, params, api_key=params.api_key or dynamic_api_key, api_base=api_base, client=client
)
if provider == LlmProviders.OPENROUTER.value
else self._adiscover_openai_compatible_model_info(
raw_deployment,
params,
model=model,
provider=provider,
api_key=params.api_key or dynamic_api_key,
api_base=api_base,
client=client,
)
)
model_id: Final = deployment.model_info.id
if (
discovered is None
or not (discovered.limits or discovered.limits_by_model)
or model_id is None
or self._core_get_model_info(model_id) is not raw_deployment
):
return
api_key: Final = await aresolve_api_key_secret_reference(
params.api_key or dynamic_api_key,
self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1)
self._discovered_model_info_cache.delete_cache(model_id)
self._discovered_model_info_cache.set_cache(model_id, discovered)
self._invalidate_model_group_info_cache()
async def _adiscover_openai_compatible_model_info(
self,
raw_deployment: Mapping[str, object],
params: LiteLLM_Params,
*,
model: str,
provider: str,
api_key: str | None,
api_base: str | None,
client: AsyncHTTPHandler | None,
) -> DiscoveredDeploymentModelInfo | None:
if provider not in MODEL_INFO_DISCOVERY_PROVIDERS or api_base is None or "*" in model:
return None
resolved_key: Final = await aresolve_api_key_secret_reference(
api_key, model=model, llm_provider=provider, upstream=MappingProxyType({"api_base": params.api_base})
)
limits: Final = await get_openai_compatible_model_info(
model=model,
llm_provider=provider,
api_base=api_base,
headers=self._model_info_request_headers(params, resolved_key),
client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI),
cache=self._core_cache.in_memory_cache,
)
return DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits)
async def _adiscover_openrouter_model_info(
self,
raw_deployment: Mapping[str, object],
params: LiteLLM_Params,
*,
api_key: str | None,
api_base: str | None,
client: AsyncHTTPHandler | None,
) -> DiscoveredDeploymentModelInfo | None:
resolved_key: Final = await aresolve_api_key_secret_reference(
OpenRouterModelInfo.get_api_key(api_key),
model=params.model,
llm_provider=LlmProviders.OPENROUTER.value,
upstream=MappingProxyType({"api_base": params.api_base}),
)
if not resolved_key:
return None
catalog: Final = await aget_openrouter_catalog_limits(
api_base=OpenRouterModelInfo.get_api_base(api_base),
headers=self._model_info_request_headers(params, resolved_key),
client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENROUTER),
cache=self._core_cache.in_memory_cache,
)
if "*" in params.model:
return DiscoveredDeploymentModelInfo(
deployment=raw_deployment, limits=MappingProxyType({}), limits_by_model=catalog
)
return DiscoveredDeploymentModelInfo(
deployment=raw_deployment, limits=openrouter_catalog_limits_for_model(catalog, params.model)
)
@staticmethod
def _model_info_request_headers(params: LiteLLM_Params, api_key: str | None) -> Mapping[str, str]:
headers: Final = TypeAdapter(Mapping[str, str]).validate_python(
params.get("extra_headers") or params.get("headers") or MappingProxyType({})
)
auth_headers: Final = (
MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({})
)
limits: Final = await get_openai_compatible_model_info(
model=model,
api_base=api_base,
headers=MappingProxyType(
{
**auth_headers,
**MappingProxyType({key.lower(): value for key, value in headers.items()}),
}
),
client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI),
cache=self._core_cache.in_memory_cache,
return MappingProxyType(
{
**auth_headers,
**MappingProxyType({key.lower(): value for key, value in headers.items()}),
}
)
model_id: Final = deployment.model_info.id
if not limits or model_id is None or self._core_get_model_info(model_id) is not raw_deployment:
return
self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1)
self._discovered_model_info_cache.delete_cache(model_id)
self._discovered_model_info_cache.set_cache(
model_id, DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits)
)
self._invalidate_model_group_info_cache()
def _core_get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
def _core_get_discovered_model_info(self, model_id: str | None, model: str | None = None) -> Mapping[str, int]:
cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id)
if (
model_id is not None
and isinstance(cached, DiscoveredDeploymentModelInfo)
and cached.deployment is self._core_get_model_info(model_id)
):
configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"])
return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None})
configured: Final = _MODEL_INFO_ADAPTER.validate_python(cached.deployment["model_info"])
limits: Final = (
openrouter_catalog_limits_for_model(cached.limits_by_model, model)
if model is not None and cached.limits_by_model
else cached.limits
)
return MappingProxyType({key: value for key, value in limits.items() if configured.get(key) is None})
return MappingProxyType({})
def _core_get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None:
@ -12296,21 +12369,22 @@ class Router(DeprecatedRouterMembers):
configured or discovered token limits. Resolved via O(1) index
lookup.
Returns None for wildcard-expanded or unknown names, where the listed name is the
real model name and no deployment-specific information exists, and treats a
malformed configured limit as absent rather than failing the listing.
Returns None for unknown names and for wildcard-expanded names whose wildcard
deployment holds no catalog row for them, where the listed name is the real model
name and no deployment-specific information exists, and treats a malformed
configured limit as absent rather than failing the listing.
The whole group is read rather than just its first deployment, so a group that
mixes models does not advertise a window that depends on config order; the widest
one is reported, which is what get_model_group_info already shows the Admin UI.
Keys are deduplicated, so the ordinary group of interchangeable deployments of one
model still costs the caller a single cost-map lookup. Unlike get_model_group_info,
this never triggers pattern matching or deep copies, so it is safe to call per
listed model on the /v1/models hot path.
this never deep copies a deployment, so it is safe to call per listed model on the
/v1/models hot path.
"""
deployments: Final = self._core_deployments_named(model_name)
if not deployments:
return None
return self._core_get_wildcard_listing_info(model_name)
model_infos: Final = tuple(
MappingProxyType(
@ -12338,6 +12412,26 @@ class Router(DeprecatedRouterMembers):
max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
)
def _core_get_wildcard_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None:
model_infos: Final = tuple(self._wildcard_catalog_model_infos(model_name))
if not model_infos:
return None
return DeploymentModelListingInfo(
cost_map_keys=(),
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),
max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
)
def _wildcard_catalog_model_infos(self, model_name: str) -> Iterator[Mapping[str, object]]:
for deployment, routed_model in self._core_pattern_router.get_routed_deployments(model_name):
configured = _configured_model_info(deployment)
model_id = configured.get("id")
discovered = self._core_get_discovered_model_info(
model_id if isinstance(model_id, str) else None, routed_model
)
if discovered:
yield MappingProxyType({**configured, **discovered})
@staticmethod
def _widest_configured_limit(model_infos: Sequence[Mapping[str, object]], field: str) -> int | None:
"""The largest usable value of ``field`` across a group's configured model_info blocks."""
@ -12776,7 +12870,7 @@ class Router(DeprecatedRouterMembers):
custom_model_info = (
{
**copy.deepcopy(litellm.model_cost.get(model_id) or MappingProxyType({})),
**self._core_get_discovered_model_info(model_id),
**self._core_get_discovered_model_info(model_id, model_name),
}
if model_id in litellm.model_cost
else None

9
litellm/router_api.py
View file

@ -247,9 +247,12 @@ class RouterRegistry:
id=id, # pyright: ignore[reportArgumentType] # see above
)
def discovered_model_info(self, id: str | None) -> Mapping[str, int]:
"""Limits discovered from the provider for a deployment (was ``get_discovered_model_info``)."""
return self._router._core_get_discovered_model_info(id)
def discovered_model_info(self, id: str | None, model: str | None = None) -> Mapping[str, int]:
"""Limits discovered from the provider for a deployment (was ``get_discovered_model_info``).
``model`` is the concrete model a wildcard deployment serves, which selects its row in the catalog.
"""
return self._router._core_get_discovered_model_info(id, model)
def zero_cost(self) -> Mapping[str, bool] | None:
"""The per-deployment zero-cost lookup the auth checks read (was ``_zero_cost_cache``)."""

4
litellm/router_deprecated.py
View file

@ -420,9 +420,9 @@ class DeprecatedRouterMembers:
id=id, # pyright: ignore[reportArgumentType] # see above
)
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
def get_discovered_model_info(self, model_id: str | None, model: str | None = None) -> Mapping[str, int]:
_note("get_discovered_model_info", "registry().discovered_model_info(id)")
return _as_router(self)._core_get_discovered_model_info(model_id)
return _as_router(self)._core_get_discovered_model_info(model_id, model)
# -- registry: many deployments -------------------------------------------------------------
def get_model_list(

23
litellm/router_utils/pattern_match_deployments.py
View file

@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching
import copy
import itertools
import re
from collections.abc import Mapping
from re import Match
from typing import Final
@ -260,8 +261,10 @@ class PatternMatchRouter:
return pattern_match
return []
def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]:
"""The models get_deployments_by_pattern would route ``model`` to, without copying the deployments."""
def get_routed_deployments(
self, model: str, custom_llm_provider: str | None = None
) -> tuple[tuple[Mapping[str, object], str], ...]:
"""Each deployment get_deployments_by_pattern would route ``model`` to, uncopied, with its concrete model."""
provider: Final = custom_llm_provider or inferred_provider(model)
requests_by_pattern: Final = itertools.product((model, f"{provider}/{model}"), self.patterns)
matched: Final = next(
@ -273,17 +276,25 @@ class PatternMatchRouter:
None,
)
if matched is None:
return frozenset()
return ()
pattern_match, pattern = matched
return frozenset(
PatternMatchRouter.set_deployment_model_name(
matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"]
return tuple(
(
deployment,
PatternMatchRouter.set_deployment_model_name(
matched_pattern=pattern_match,
litellm_deployment_litellm_model=deployment["litellm_params"]["model"],
),
)
for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts
pattern, ()
)
)
def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]:
"""The models get_deployments_by_pattern would route ``model`` to, without copying the deployments."""
return frozenset(routed_model for _, routed_model in self.get_routed_deployments(model, custom_llm_provider))
# Example usage:
# router = PatternRouter()

1
litellm/types/router.py
View file

@ -917,6 +917,7 @@ class Deployment(LiteLLMBaseModel):
class DiscoveredDeploymentModelInfo:
deployment: Mapping[str, object]
limits: Mapping[str, int]
limits_by_model: Mapping[str, Mapping[str, int]] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)

2
tests/integration/providers/_cache_control_marks_support.py
View file

@ -464,6 +464,7 @@ def owned_config(
*,
litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}),
router_settings: Mapping[str, JsonValue] = MappingProxyType({}),
general_settings: Mapping[str, JsonValue] = MappingProxyType({}),
) -> Path:
config: Final = _JSON_OBJECT.validate_python(
yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
@ -473,6 +474,7 @@ def owned_config(
"model_list": list(model_list),
"litellm_settings": {**object_value(config["litellm_settings"]), **litellm_settings},
"router_settings": {**object_value(config["router_settings"]), "num_retries": 0, **router_settings},
"general_settings": {**object_value(config["general_settings"]), **general_settings},
}
path: Final = directory / f"cache-control-marks-{uuid.uuid4().hex}.yaml"
path.write_text(yaml.safe_dump(merged))

109
tests/integration/routing/test_openrouter_catalog_limits_wire.py Normal file
View file

@ -0,0 +1,109 @@
import json
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import (
Gateway,
eventually,
gateway_from_environment,
list_value,
object_value,
string_value,
)
from integration._support.process import graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from integration.providers._cache_control_marks_support import owned_config
from pydantic import JsonValue
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_API_KEY: Final = "integration-openrouter-key"
_CATALOG_ROUTE: Final = "/api/v1/models/user"
_CATALOG: Final = {
"data": [
{"id": "vendor/capped", "context_length": 4096, "top_provider": {"max_completion_tokens": 1024}},
{"id": "vendor/uncapped", "context_length": 8192, "top_provider": {"max_completion_tokens": None}},
]
}
_NAMED_MODEL: Final = "gpt-mini-named"
_EXPECTED_LIMITS: Final = {
_NAMED_MODEL: (4096, 1024),
"openrouter/vendor/capped": (4096, 1024),
"openrouter/vendor/uncapped": (8192, None),
}
@dataclass(frozen=True, slots=True)
class _CatalogProxy:
gateway: Gateway
wire: Wire
def _catalog_reply(_request: Request) -> Reply:
return Reply(body=json.dumps(_CATALOG).encode())
def _deployment(model_name: str, model: str, wire: Wire) -> JsonValue:
return {
"model_name": model_name,
"litellm_params": {"model": model, "api_key": _API_KEY, "api_base": f"{wire.url}/api/v1"},
}
@pytest.fixture(scope="module")
def catalog_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_CatalogProxy]:
directory: Final[Path] = tmp_path_factory.mktemp("openrouter-catalog-limits")
with gateway_from_environment() as gateway, wire_server(_catalog_reply) as wire:
config: Final = owned_config(
directory,
[
_deployment(_NAMED_MODEL, "openrouter/vendor/capped", wire),
_deployment("openrouter/*", "openrouter/*", wire),
],
litellm_settings={"check_provider_endpoint": True},
general_settings={"disable_model_info_refresh": False},
)
with owned_proxy_process(gateway, directory, {}, config=config) as owned:
yield _CatalogProxy(gateway=owned.gateway, wire=wire)
def _listed_limits(
gateway: Gateway, path: str, name_key: str, row_limits: str | None
) -> Mapping[str, tuple[JsonValue, JsonValue]]:
rows: Final = tuple(object_value(row) for row in list_value(gateway.get(path)["data"]))
named_rows: Final = {string_value(row[name_key]): row for row in rows}
limit_rows: Final = {
name: object_value(row[row_limits]) if row_limits is not None else row for name, row in named_rows.items()
}
return {
name: (limit_rows[name].get("max_input_tokens"), limit_rows[name].get("max_output_tokens"))
for name in _EXPECTED_LIMITS
if name in limit_rows
}
@pytest.mark.parametrize(
("path", "name_key", "row_limits"),
(
("/v1/model/info", "model_name", "model_info"),
("/model_group/info", "model_group", None),
("/v1/models", "id", None),
),
)
def test_openrouter_catalog_limits_reach_the_model_info_routes_with_no_cost_map_row(
catalog_proxy: _CatalogProxy, path: str, name_key: str, row_limits: str | None
) -> None:
limits: Final = eventually(
lambda: _listed_limits(catalog_proxy.gateway, path, name_key, row_limits),
lambda observed: observed == _EXPECTED_LIMITS,
seconds=30,
)
assert limits == _EXPECTED_LIMITS
catalog_calls: Final = catalog_proxy.wire.drain()
assert {(call.method, call.target, call.headers["authorization"]) for call in catalog_calls} <= {
("GET", _CATALOG_ROUTE, f"Bearer {_API_KEY}")
}

16
tests/unit/llms/openai_like/test_model_info.py
View file

@ -124,3 +124,19 @@ async def test_unavailable_metadata_is_best_effort_and_negative_cached(
== {}
)
assert responder.call_count == 1
async def test_first_card_wins_when_an_endpoint_repeats_a_model_id() -> None:
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200, json={"data": [{"id": "model", "max_model_len": 4096}, {"id": "model", "max_model_len": 8192}]}
)
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
handler.client = client
limits: Final = await get_openai_compatible_model_info(
model="model", api_base="https://backend.test", headers={}, client=handler, cache=InMemoryCache()
)
assert limits == {"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096}

113
tests/unit/llms/openrouter/test_openrouter_common_utils.py
View file

@ -1,11 +1,19 @@
from collections.abc import Mapping
from typing import Final
from unittest.mock import Mock
import httpx
import pytest
import respx
import litellm
from litellm.llms.openrouter.common_utils import OpenRouterModelInfo
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.openrouter.common_utils import (
OpenRouterModelInfo,
aget_openrouter_catalog_limits,
openrouter_catalog_limits_for_model,
)
DEFAULT_BASE: Final = "https://openrouter.ai/api/v1"
KEY_SOURCES: Final = ("api_key", "litellm.api_key", "litellm.openrouter_key", "OPENROUTER_API_KEY", "OR_API_KEY")
@ -157,3 +165,106 @@ def test_validate_environment_without_any_key_sends_no_authorization():
)
def test_get_base_model_drops_only_the_openrouter_prefix(model: str, base_model: str):
assert OpenRouterModelInfo.get_base_model(model) == base_model
@respx.mock
def test_get_models_keeps_listing_when_a_catalog_row_carries_unusable_limits():
respx.get(f"{DEFAULT_BASE}/models/user").mock(
return_value=httpx.Response(
200,
json={
"data": [
{"id": "vendor/junk-limits", "context_length": "many", "top_provider": "none"},
{"id": "vendor/junk-cap", "top_provider": {"max_completion_tokens": "many"}},
{"id": "vendor/no-limits"},
]
},
)
)
assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [
"openrouter/vendor/junk-limits",
"openrouter/vendor/junk-cap",
"openrouter/vendor/no-limits",
]
@pytest.mark.parametrize(
("row", "expected"),
(
(
{"context_length": 4096, "top_provider": {"max_completion_tokens": 1024}},
{"max_tokens": 1024, "max_input_tokens": 4096, "max_output_tokens": 1024},
),
(
{"context_length": 4096, "top_provider": {"max_completion_tokens": None}},
{"max_tokens": 4096, "max_input_tokens": 4096},
),
({"context_length": 4096}, {"max_tokens": 4096, "max_input_tokens": 4096}),
(
{"context_length": 4096, "top_provider": {"context_length": 8192, "max_completion_tokens": 1024}},
{"max_tokens": 1024, "max_input_tokens": 4096, "max_output_tokens": 1024},
),
({"top_provider": {"max_completion_tokens": 1024}}, {"max_tokens": 1024, "max_output_tokens": 1024}),
({"context_length": 0, "top_provider": {"max_completion_tokens": True}}, {}),
({"context_length": "many", "top_provider": "none"}, {}),
),
)
async def test_catalog_row_becomes_the_published_limits_and_never_invents_an_output_cap(
row: Mapping[str, object], expected: Mapping[str, int]
) -> None:
def respond(request: httpx.Request) -> httpx.Response:
assert str(request.url) == f"{DEFAULT_BASE}/models/user"
assert request.headers["authorization"] == "Bearer sk-or-test"
return httpx.Response(200, json={"data": [{"id": "vendor/model", **row}]})
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
handler.client = client
catalog: Final = await aget_openrouter_catalog_limits(
api_base=DEFAULT_BASE,
headers={"Authorization": "Bearer sk-or-test"},
client=handler,
cache=InMemoryCache(),
)
assert catalog == {"vendor/model": expected}
async def test_a_failed_catalog_fetch_is_cached_so_the_next_refresh_does_not_call_again() -> None:
responder: Final = Mock(return_value=httpx.Response(503))
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
handler.client = client
cache: Final = InMemoryCache()
for _ in range(2):
assert (
await aget_openrouter_catalog_limits(api_base=DEFAULT_BASE, headers={}, client=handler, cache=cache)
== {}
)
assert responder.call_count == 1
@pytest.mark.parametrize(
("model", "expected"),
(
("openrouter/vendor/model", {"max_input_tokens": 1}),
("vendor/model", {"max_input_tokens": 1}),
("openrouter/vendor/model:nitro", {"max_input_tokens": 1}),
("openrouter/vendor/model:free", {"max_input_tokens": 2}),
("openrouter/auto", {"max_input_tokens": 3}),
("openrouter/vendor/unlisted", {}),
("openrouter/vendor/unlisted:nitro", {}),
),
)
def test_catalog_lookup_finds_a_model_by_its_id_then_by_the_id_without_its_variant(
model: str, expected: Mapping[str, int]
) -> None:
catalog: Final = {
"vendor/model": {"max_input_tokens": 1},
"vendor/model:free": {"max_input_tokens": 2},
"openrouter/auto": {"max_input_tokens": 3},
}
assert openrouter_catalog_limits_for_model(catalog, model) == expected

88
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import respx
from fastapi.testclient import TestClient
import litellm
@ -165,6 +166,93 @@ async def test_upstream_limits_reach_model_info_routes(
_invalidate_model_cost_lowercase_map()
async def test_openrouter_catalog_limits_reach_every_model_info_route(
client: TestClient,
auth_as: Callable[[], AbstractContextManager[object]],
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
router: Final = litellm.Router(
model_list=[
{
"model_name": "named",
"litellm_params": {"model": "openrouter/vendor/capped", "api_key": "openrouter-key"},
"model_info": {"id": "named-deployment"},
},
{
"model_name": "overridden",
"litellm_params": {"model": "openrouter/vendor/capped", "api_key": "openrouter-key"},
"model_info": {"id": "overridden-deployment", "max_output_tokens": 512},
},
{
"model_name": "openrouter/*",
"litellm_params": {"model": "openrouter/*", "api_key": "openrouter-key"},
"model_info": {"id": "wildcard-deployment"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
monkeypatch.setattr(proxy_server, "user_model", None)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
catalog: Final = {
"data": [
{"id": "vendor/capped", "context_length": 4096, "top_provider": {"max_completion_tokens": 1024}},
{"id": "vendor/uncapped", "context_length": 8192, "top_provider": {"max_completion_tokens": None}},
]
}
expected: Final = {
"named": (4096, 1024),
"overridden": (4096, 512),
"openrouter/vendor/capped": (4096, 1024),
"openrouter/vendor/uncapped": (8192, None),
}
def limits_by_name(rows: list[dict[str, object]], name_key: str) -> dict[object, tuple[object, object]]:
return {
row[name_key]: (row.get("max_input_tokens"), row.get("max_output_tokens"))
for row in rows
if row[name_key] in expected
}
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(
transport=httpx.MockTransport(lambda request: httpx.Response(200, json=catalog))
) as upstream:
handler.client = upstream
litellm.in_memory_llm_clients_cache.set_cache("async_httpx_clientopenrouter", handler)
await proxy_server.ProxyStartupEvent.refresh_model_info()
with respx.mock(assert_all_called=False) as sync_upstream, auth_as():
sync_upstream.get("https://openrouter.ai/api/v1/models/user").mock(
return_value=httpx.Response(200, json=catalog)
)
for path in ("/v1/model/info", "/model/info"):
info_response: Final = client.get(path)
assert info_response.status_code == 200, info_response.text
assert (
limits_by_name(
[
{"model_name": row["model_name"], **row["model_info"]}
for row in info_response.json()["data"]
],
"model_name",
)
== expected
)
group_response: Final = client.get("/model_group/info")
assert group_response.status_code == 200, group_response.text
assert limits_by_name(group_response.json()["data"], "model_group") == expected
models_response: Final = client.get("/v1/models")
assert models_response.status_code == 200, models_response.text
assert limits_by_name(models_response.json()["data"], "id") == expected
_invalidate_model_cost_lowercase_map()
# ---------------------------------------------------------------------------
# GET /v2/model/info
# ---------------------------------------------------------------------------

2
tests/unit/proxy/router_entry_point_stubs.py
View file

@ -142,7 +142,7 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble:
registry.deployment.side_effect = lambda id: router.get_deployment(model_id=id)
registry.deployment_dict.side_effect = lambda id: router.get_model_info(id=id)
registry.has_deployment.side_effect = lambda id: router.has_model_id(id)
registry.discovered_model_info.side_effect = lambda id: router.get_discovered_model_info(id)
registry.discovered_model_info.side_effect = lambda id, model=None: router.get_discovered_model_info(id, model)
def _credentials(id, *, form="params", **team):
# Old call sites passed team_id (even None) exactly when the new ones do, so pass it on as given.

2
tests/unit/test_router/test_router_entry_points.py
View file

@ -585,7 +585,7 @@ FORWARDS = [
"_core_get_router_model_info",
((), {"deployment": None, "received_model_name": "gpt", "id": "dep-a"}),
),
(lambda r: r.registry().discovered_model_info("dep-a"), "_core_get_discovered_model_info", (("dep-a",), {})),
(lambda r: r.registry().discovered_model_info("dep-a"), "_core_get_discovered_model_info", (("dep-a", None), {})),
(
lambda r: r.registry().deployments("gpt", source="routing_groups"),
"_core_get_model_list_from_routing_groups",

347
tests/unit/test_router_model_cost_isolation.py
View file

@ -19,6 +19,7 @@ import re
import sys
import threading
from pathlib import Path
from collections.abc import Callable, Iterator
from types import MappingProxyType
from typing import Final, cast
from unittest.mock import Mock, patch
@ -37,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
from litellm.types.router import (
Deployment,
DeploymentModelListingInfo,
DeploymentTypedDict,
LiteLLM_Params,
LiteLLMParamsTypedDict,
@ -456,6 +458,351 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa
_invalidate_model_cost_lowercase_map()
@pytest.fixture
def openrouter_settings_cleared(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
for env_var in ("OPENROUTER_API_BASE", "OPENROUTER_API_KEY", "OR_API_KEY"):
monkeypatch.delenv(env_var, raising=False)
monkeypatch.setattr(litellm, "api_key", None)
monkeypatch.setattr(litellm, "api_base", None)
monkeypatch.setattr(litellm, "openrouter_key", None)
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
yield
_invalidate_model_cost_lowercase_map()
def _openrouter_catalog_response(request: httpx.Request) -> httpx.Response:
assert request.url.path.endswith("/models/user")
return httpx.Response(
200,
json={
"data": [
{"id": "vendor/capped", "context_length": 4096, "top_provider": {"max_completion_tokens": 1024}},
{"id": "vendor/uncapped", "context_length": 8192, "top_provider": {"max_completion_tokens": None}},
]
},
)
def _openrouter_deployment(
model_name: str, model: str, deployment_id: str, **model_info: int | None
) -> DeploymentTypedDict:
return {
"model_name": model_name,
"litellm_params": {"model": model, "api_key": "openrouter-key"},
"model_info": {"id": deployment_id, **model_info},
}
async def _refresh_against(router: Router, respond: Callable[[httpx.Request], httpx.Response]) -> None:
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
handler.client = client
await router.arefresh_model_info(client=handler)
def _group_limits(router: Router, model_group: str) -> tuple[int | None, int | None]:
group: Final = router.get_model_group_info(model_group)
assert group is not None
return group.max_input_tokens, group.max_output_tokens
PUBLISHED_OPENROUTER_LIMITS: Final = (
("vendor/capped", (4096, 1024)),
("vendor/capped:nitro", (4096, 1024)),
("vendor/uncapped", (8192, None)),
("vendor/unlisted", (None, None)),
)
@pytest.mark.parametrize(("catalog_id", "expected"), PUBLISHED_OPENROUTER_LIMITS)
async def test_openrouter_named_deployment_shows_the_limits_the_catalog_publishes(
catalog_id: str, expected: tuple[int | None, int | None], openrouter_settings_cleared: None
) -> None:
router: Final = Router(model_list=[_openrouter_deployment("named", f"openrouter/{catalog_id}", "named-id")])
await _refresh_against(router, _openrouter_catalog_response)
assert _group_limits(router, "named") == expected
@pytest.mark.parametrize(("catalog_id", "expected"), PUBLISHED_OPENROUTER_LIMITS)
async def test_openrouter_wildcard_deployment_shows_the_limits_of_each_concrete_model(
catalog_id: str, expected: tuple[int | None, int | None], openrouter_settings_cleared: None
) -> None:
router: Final = Router(model_list=[_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id")])
await _refresh_against(router, _openrouter_catalog_response)
assert _group_limits(router, f"openrouter/{catalog_id}") == expected
async def test_discovered_model_info_without_a_model_keeps_answering_for_the_deployment_itself(
openrouter_settings_cleared: None,
) -> None:
router: Final = Router(
model_list=[
_openrouter_deployment("named", "openrouter/vendor/capped", "named-id"),
_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id"),
]
)
await _refresh_against(router, _openrouter_catalog_response)
own_limits: Final = {"max_tokens": 1024, "max_input_tokens": 4096, "max_output_tokens": 1024}
registry: Final = router.registry()
assert registry.discovered_model_info("named-id") == own_limits
assert registry.discovered_model_info("named-id", "openrouter/vendor/uncapped") == own_limits
assert registry.discovered_model_info("wild-id") == {}
assert registry.discovered_model_info("wild-id", "openrouter/vendor/capped") == own_limits
async def test_operator_model_info_beats_the_openrouter_catalog(openrouter_settings_cleared: None) -> None:
router: Final = Router(
model_list=[
_openrouter_deployment("named", "openrouter/vendor/capped", "named-id", max_input_tokens=100),
_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id", max_output_tokens=50),
]
)
await _refresh_against(router, _openrouter_catalog_response)
assert _group_limits(router, "named") == (100, 1024)
assert _group_limits(router, "openrouter/vendor/capped") == (4096, 50)
@pytest.mark.parametrize(
("deployment", "model_group"),
(
(("named", "openrouter/vendor/capped", "named-id"), "named"),
(("openrouter/*", "openrouter/*", "wild-id"), "openrouter/vendor/capped"),
),
)
async def test_openrouter_catalog_beats_a_cost_map_row_for_a_listed_model(
deployment: tuple[str, str, str], model_group: str, openrouter_settings_cleared: None
) -> None:
litellm.model_cost["openrouter/vendor/capped"] = {
"litellm_provider": "openrouter",
"mode": "chat",
"max_input_tokens": 1,
"max_output_tokens": 2,
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
_invalidate_model_cost_lowercase_map()
router: Final = Router(model_list=[_openrouter_deployment(*deployment)])
await _refresh_against(router, _openrouter_catalog_response)
assert _group_limits(router, model_group) == (4096, 1024)
@pytest.mark.parametrize(
("deployment", "model_group"),
(
(("named", "openrouter/vendor/unlisted", "named-id"), "named"),
(("openrouter/*", "openrouter/*", "wild-id"), "openrouter/vendor/unlisted"),
),
)
async def test_openrouter_cost_map_row_stays_for_a_model_the_catalog_does_not_list(
deployment: tuple[str, str, str], model_group: str, openrouter_settings_cleared: None
) -> None:
litellm.model_cost["openrouter/vendor/unlisted"] = {
"litellm_provider": "openrouter",
"mode": "chat",
"max_input_tokens": 1,
"max_output_tokens": 2,
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
_invalidate_model_cost_lowercase_map()
router: Final = Router(model_list=[_openrouter_deployment(*deployment)])
await _refresh_against(router, _openrouter_catalog_response)
assert _group_limits(router, model_group) == (1, 2)
async def test_openrouter_group_takes_the_widest_published_limit_with_no_cost_data(
openrouter_settings_cleared: None,
) -> None:
router: Final = Router(
model_list=[
_openrouter_deployment("group", "openrouter/vendor/capped", "capped-id"),
_openrouter_deployment("group", "openrouter/vendor/uncapped", "uncapped-id"),
]
)
await _refresh_against(router, _openrouter_catalog_response)
assert _group_limits(router, "group") == (8192, 1024)
async def test_openrouter_deployments_on_one_key_share_one_catalog_fetch(openrouter_settings_cleared: None) -> None:
router: Final = Router(
model_list=[
_openrouter_deployment("first", "openrouter/vendor/capped", "first-id"),
_openrouter_deployment("second", "openrouter/vendor/uncapped", "second-id"),
_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id"),
]
)
responder: Final = Mock(side_effect=_openrouter_catalog_response)
await _refresh_against(router, responder)
assert responder.call_count == 1
assert _group_limits(router, "first") == (4096, 1024)
assert _group_limits(router, "second") == (8192, None)
assert _group_limits(router, "openrouter/vendor/capped") == (4096, 1024)
@pytest.mark.parametrize(
("deployment", "model_group"),
(
(("named", "openrouter/vendor/capped", "named-id"), "named"),
(("openrouter/*", "openrouter/*", "wild-id"), "openrouter/vendor/capped"),
),
)
async def test_an_openrouter_catalog_outage_keeps_the_last_published_limits(
deployment: tuple[str, str, str], model_group: str, openrouter_settings_cleared: None
) -> None:
router: Final = Router(model_list=[_openrouter_deployment(*deployment)])
await _refresh_against(router, _openrouter_catalog_response)
router.cache.in_memory_cache.flush_cache()
await _refresh_against(router, lambda request: httpx.Response(503))
assert _group_limits(router, model_group) == (4096, 1024)
async def test_openrouter_discovery_without_a_key_makes_no_call(openrouter_settings_cleared: None) -> None:
router: Final = Router(
model_list=[
{"model_name": "named", "litellm_params": {"model": "openrouter/vendor/capped"}},
{"model_name": "openrouter/*", "litellm_params": {"model": "openrouter/*"}},
]
)
responder: Final = Mock(side_effect=_openrouter_catalog_response)
await _refresh_against(router, responder)
assert responder.call_count == 0
assert _group_limits(router, "named") == (None, None)
async def test_openrouter_discovery_skips_a_caller_credentials_deployment(
openrouter_settings_cleared: None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("OPENROUTER_API_KEY", "environment-key")
router: Final = Router(
model_list=[
{
"model_name": "byok",
"litellm_params": {"model": "openrouter/vendor/capped", "use_clientside_credentials": True},
}
]
)
responder: Final = Mock(side_effect=_openrouter_catalog_response)
await _refresh_against(router, responder)
assert responder.call_count == 0
assert _group_limits(router, "byok") == (None, None)
async def test_openrouter_discovery_sends_the_secret_behind_an_oidc_key_to_the_configured_base(
openrouter_settings_cleared: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path))
token: Final = tmp_path / "token"
token.write_text("openrouter-secret")
router: Final = Router(
model_list=[
{
"model_name": "named",
"litellm_params": {
"model": "openrouter/vendor/capped",
"api_base": "https://openrouter.test/tenant/v1",
"api_key": f"oidc/file/{token}",
},
}
]
)
requests: Final[list[httpx.Request]] = []
def respond(request: httpx.Request) -> httpx.Response:
requests.append(request)
return _openrouter_catalog_response(request)
await _refresh_against(router, respond)
assert [str(request.url) for request in requests] == ["https://openrouter.test/tenant/v1/models/user"]
assert requests[0].headers["authorization"] == "Bearer openrouter-secret"
assert _group_limits(router, "named") == (4096, 1024)
async def test_openrouter_discovery_uses_the_key_from_the_environment(
openrouter_settings_cleared: None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("OPENROUTER_API_KEY", "environment-key")
router: Final = Router(
model_list=[{"model_name": "named", "litellm_params": {"model": "openrouter/vendor/capped"}}]
)
authorizations: Final[list[str | None]] = []
def respond(request: httpx.Request) -> httpx.Response:
authorizations.append(request.headers.get("Authorization"))
return _openrouter_catalog_response(request)
await _refresh_against(router, respond)
assert authorizations == ["Bearer environment-key"]
async def test_a_wildcard_on_another_provider_is_still_not_discovered(openrouter_settings_cleared: None) -> None:
router: Final = Router(
model_list=[
{
"model_name": "hosted/*",
"litellm_params": {
"model": "hosted_vllm/*",
"api_base": "https://backend.test/v1",
"api_key": "local-key",
},
}
]
)
responder: Final = Mock(return_value=httpx.Response(200, json={"data": []}))
await _refresh_against(router, responder)
assert responder.call_count == 0
assert router.registry().group("hosted/model", field="listing_info") is None
async def test_openrouter_wildcard_listing_info_shows_each_concrete_models_limits(
openrouter_settings_cleared: None,
) -> None:
router: Final = Router(model_list=[_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id")])
await _refresh_against(router, _openrouter_catalog_response)
listing_info: Final = router.registry().group("openrouter/vendor/capped", field="listing_info")
assert listing_info == DeploymentModelListingInfo(cost_map_keys=(), max_input_tokens=4096, max_output_tokens=1024)
assert router.registry().group("openrouter/vendor/unlisted", field="listing_info") is None
async def test_openrouter_wildcard_listing_info_keeps_the_operator_limit(openrouter_settings_cleared: None) -> None:
router: Final = Router(
model_list=[_openrouter_deployment("openrouter/*", "openrouter/*", "wild-id", max_output_tokens=50)]
)
await _refresh_against(router, _openrouter_catalog_response)
listing_info: Final = router.registry().group("openrouter/vendor/capped", field="listing_info")
assert listing_info == DeploymentModelListingInfo(cost_map_keys=(), max_input_tokens=4096, max_output_tokens=50)
def test_should_not_pollute_shared_key_with_zero_cost_pricing():
"""
When deployment A has input_cost_per_token=0 and deployment B has no