From da38062124fc42538c17d84c933f552f9555c0a8 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 5 Oct 2026 14:14:27 +0200 Subject: [PATCH 01/34] feat(proxy): advertise provider model metadata to gateway clients --- ci_cd/generate_model_prices_schema.py | 11 +- examples/model_metadata/README.md | 42 +++ litellm/llms/chatgpt/authenticator.py | 8 +- litellm/llms/chatgpt/model_info.py | 147 +++++++++ litellm/llms/openai_like/model_info.py | 127 ++++++-- litellm/proxy/_types.py | 5 +- litellm/proxy/model_discovery.py | 59 ++++ litellm/proxy/proxy_server.py | 24 +- litellm/proxy/utils.py | 52 ++- litellm/router.py | 81 +++-- litellm/types/proxy/model_listing.py | 18 +- litellm/types/proxy/model_metadata.py | 85 +++++ litellm/types/router.py | 9 +- litellm/types/utils.py | 5 +- litellm/utils.py | 3 + model_prices_and_context_window.schema.json | 10 +- tests/unit/llms/chatgpt/test_model_info.py | 168 ++++++++++ .../unit/llms/openai_like/test_model_info.py | 89 +++++- .../proxy_server/test_routes_model_info.py | 5 +- tests/unit/proxy/test_model_discovery.py | 67 ++++ ..._utils_model_creation_and_error_logging.py | 63 ++++ tests/unit/test_model_prices_schema.py | 26 ++ .../unit/test_router_model_cost_isolation.py | 298 ++++++++++++------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 40 +++ 24 files changed, 1272 insertions(+), 170 deletions(-) create mode 100644 examples/model_metadata/README.md create mode 100644 litellm/llms/chatgpt/model_info.py create mode 100644 litellm/proxy/model_discovery.py create mode 100644 litellm/types/proxy/model_metadata.py create mode 100644 tests/unit/llms/chatgpt/test_model_info.py create mode 100644 tests/unit/proxy/test_model_discovery.py diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 8eec07dadda..35107c595d2 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -169,13 +169,18 @@ ARRAY_KEYS: dict[str, JsonSchema] = { } INTEGER_KEYS: dict[str, JsonSchema] = { + "context_window": { + "type": "integer", + "minimum": 1, + "description": "Maximum combined input and generated output tokens, independent of the input and output limits.", + }, "max_tokens": { **NONNEG_INTEGER, "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.", }, "max_input_tokens": { **NONNEG_INTEGER, - "description": "Maximum prompt/context tokens the model accepts.", + "description": "Maximum input tokens, independent of the combined input and output context window.", }, "max_output_tokens": { **NONNEG_INTEGER, @@ -280,7 +285,7 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: "Gates whether a non-default temperature or the top_p/logprobs sampling params are " "accepted, which hold only when the effort resolves to 'none'." ), - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], }, "comment": STRING, "audio_transcription_config": STRING, @@ -304,7 +309,7 @@ def classify(key: str, modes: tuple) -> Optional[JsonSchema]: def build_schema(prices: dict) -> JsonSchema: entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS} - all_keys = tuple(sorted({key for entry in entries.values() for key in entry})) + all_keys = tuple(sorted({key for entry in entries.values() for key in entry} | {"context_window"})) modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry})) unclassified = tuple(key for key in all_keys if classify(key, modes) is None) if unclassified: diff --git a/examples/model_metadata/README.md b/examples/model_metadata/README.md new file mode 100644 index 00000000000..df12d30f30d --- /dev/null +++ b/examples/model_metadata/README.md @@ -0,0 +1,42 @@ +# Gateway model discovery + +LiteLLM advertises model limits and capabilities in its normal `/v1/models` response and detailed `/model/info` response + +The shared metadata contract distinguishes `context_window`, the total combined input and generated output limit, from `max_input_tokens` and `max_output_tokens`. Unknown values are omitted. LiteLLM does not add the two maxima to derive context, or use a context value as an output limit + +Configure the models you want to expose in `model_list`. LiteLLM retrieves metadata for those models from supported upstream catalogs at startup, then refreshes every five minutes. OpenAI-compatible servers can publish the shared fields, `context_length`, or `max_model_len`. OpenRouter and Vercel catalogs are normalized from their provider-specific fields. ChatGPT uses the active account's authenticated native model catalog without starting a device login + +Upstream metadata overrides LiteLLM's bundled model catalog. Explicit `model_info` values override upstream metadata. A server that publishes only model IDs still works, and its missing limits can be configured manually in LiteLLM + +```yaml +model_list: + - model_name: shared-model + litellm_params: + model: openai/provider-model + api_base: https://inference.example/v1 + api_key: os.environ/PROVIDER_API_KEY + model_info: + context_window: 120000 + max_input_tokens: 110000 + max_output_tokens: 20000 + supports_function_calling: true + supports_parallel_function_calling: false + supports_reasoning: true + reasoning_effort_levels: [low, high, xhigh] + default_reasoning_effort: high + supported_endpoints: [/v1/chat/completions] + supported_modalities: [text, image] + supported_output_modalities: [text] +``` + +These example limits describe independent constraints. With 20000 output tokens reserved, this example allows at most 100000 input tokens because the total context limit also applies + +Reasoning levels are the exact values the configured route accepts. The list is not restricted to a fixed set of vendor names, and a default outside a known list is omitted. Explicit `false` capabilities and empty lists are preserved + +ChatGPT's native catalog supplies account-specific reasoning choices, defaults, input modalities, and parallel tool support. Its client policy windows remain separate `codex_context_window` and `codex_max_context_window` diagnostics in detailed model information. They are not advertised as physical total context or input limits. Missing canonical limits require verified catalog values or a manual `model_info` definition + +Native Codex UI reasoning aliases are normalized to API effort values. `ultra` is not advertised as a wire effort, and `persistent` maps to `disabled` + +For a model group with an explicit total context contract, discovery advertises the smallest known input, output, and total limit across its deployments. A limit is omitted if any deployment does not declare it. Capabilities require agreement across deployments, supported lists are intersected, and a default must agree across deployments. Existing input/output aggregation is preserved for legacy groups without a total context field + +Only configured models are exposed. Fetching a supplier's catalog does not add its other models to the downstream listing. Set `general_settings.disable_model_info_refresh: true` to disable upstream catalog polling diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index 563826c2b93..20c625fabcb 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -52,7 +52,7 @@ class Authenticator: def get_api_base(self) -> str: return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE - def get_access_token(self) -> str: + def get_access_token(self, *, allow_device_login: bool = True) -> str: auth_data: Final = self._read_auth_file() if auth_data: access_token: Final = _optional_str(auth_data.get("access_token")) @@ -66,6 +66,12 @@ class Authenticator: except RefreshAccessTokenError as exc: verbose_logger.warning("ChatGPT refresh token failed, re-login required: %s", exc) + if not allow_device_login: + raise GetAccessTokenError( + status_code=401, + message="ChatGPT authorization is unavailable; model discovery cannot start device login", + ) + cooldown_remaining: Final = self._get_device_code_cooldown_remaining(auth_data) if cooldown_remaining > 0: token: Final = self._wait_for_access_token(cooldown_remaining) diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py new file mode 100644 index 00000000000..36bba20c7a5 --- /dev/null +++ b/litellm/llms/chatgpt/model_info.py @@ -0,0 +1,147 @@ +import asyncio +import hashlib +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict, field_validator + +from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.types.proxy.model_metadata import GatewayModelMetadata + +from .authenticator import Authenticator +from .common_utils import get_chatgpt_default_headers + +_CATALOG_CLIENT_VERSION: Final = "0.159.3" +_EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) + + +class _ReasoningLevel(BaseModel): + effort: str + + +class _ChatGPTModel(GatewayModelMetadata): + slug: str + max_context_window: int | None = None + default_reasoning_level: str | None = None + supported_reasoning_levels: tuple[_ReasoningLevel, ...] | None = None + input_modalities: Sequence[str] | None = None + supports_parallel_tool_calls: bool | None = None + multi_agent_reasoning_effort: str | None = None + + @field_validator("max_context_window", mode="before") + @classmethod + def maximum_context(cls, value: object) -> int | None: + return GatewayModelMetadata.model_validate({"context_window": value}).context_window + + def metadata(self) -> Mapping[str, object]: + maximum: Final = GatewayModelMetadata(context_window=self.max_context_window).context_window + native_efforts: Final = ( + self.reasoning_effort_levels + if self.reasoning_effort_levels is not None + else tuple(level.effort for level in self.supported_reasoning_levels) + if self.supported_reasoning_levels is not None + else None + ) + # Codex Ultra is a multi-agent UI choice, not an API reasoning effort. + api_efforts: Final = ( + tuple("disabled" if effort == "persistent" else effort for effort in native_efforts if effort != "ultra") + if native_efforts is not None + else None + ) + native_default: Final = self.default_reasoning_effort or self.default_reasoning_level + default_effort: Final = ( + self.multi_agent_reasoning_effort + if native_default == "ultra" and self.multi_agent_reasoning_effort in (api_efforts or ()) + else "max" + if native_default == "ultra" and "max" in (api_efforts or ()) + else api_efforts[-1] + if native_default == "ultra" and api_efforts + else "disabled" + if native_default == "persistent" + else native_default + ) + normalized: Final = GatewayModelMetadata( + max_input_tokens=self.max_input_tokens, + max_output_tokens=self.max_output_tokens, + supports_function_calling=self.supports_function_calling, + supports_parallel_function_calling=( + self.supports_parallel_function_calling + if self.supports_parallel_function_calling is not None + else self.supports_parallel_tool_calls + ), + supports_reasoning=( + self.supports_reasoning + if self.supports_reasoning is not None + else bool(self.supported_reasoning_levels) + if self.supported_reasoning_levels is not None + else None + ), + reasoning_effort_levels=api_efforts, + default_reasoning_effort=default_effort, + supported_endpoints=("/responses",), + supported_modalities=( + self.supported_modalities if self.supported_modalities is not None else self.input_modalities + ), + supported_output_modalities=self.supported_output_modalities, + ) + native_windows: Final = { + "codex_context_window": self.context_window, + "codex_max_context_window": maximum, + } + return MappingProxyType( + { + **normalized.model_dump(exclude_none=True), + **{key: value for key, value in native_windows.items() if value is not None}, + } + ) + + +class _ChatGPTCatalog(BaseModel): + model_config = ConfigDict(frozen=True) + + models: tuple[_ChatGPTModel, ...] = () + + +async def get_chatgpt_model_info( + *, + model: str, + client: AsyncHTTPHandler, + cache: InMemoryCache, + api_base: str | None = None, + authenticator: Authenticator | None = None, +) -> Mapping[str, object]: + auth: Final = authenticator if authenticator is not None else Authenticator() + try: + token: Final = await asyncio.to_thread(auth.get_access_token, allow_device_login=False) + account: Final = await asyncio.to_thread(auth.get_account_id) + url: Final = f"{(api_base or auth.get_api_base()).rstrip('/')}/models?client_version={_CATALOG_CLIENT_VERSION}" + headers: Final = { + **get_chatgpt_default_headers(token, account), + "accept": "application/json", + } + cache_key: Final = ( + "chatgpt_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() + ) + cached: Final[object] = cache.get_cache(cache_key) + if isinstance(cached, _ChatGPTCatalog): + return next((card.metadata() for card in cached.models if card.slug == model), _EMPTY_METADATA) + response: Final = await client.get( + url=url, + headers=headers, + timeout=httpx.Timeout(5.0), + follow_redirects=False, + max_response_bytes=2 * 1024 * 1024, + ) + response.raise_for_status() + catalog: Final = _ChatGPTCatalog.model_validate_json(response.content) + cache.set_cache(cache_key, catalog, ttl=MODEL_INFO_REFRESH_SECONDS) + return next((card.metadata() for card in catalog.models if card.slug == model), _EMPTY_METADATA) + except Exception: # noqa: BLE001 # optional metadata discovery must not interrupt proxy startup + verbose_logger.debug("Could not discover the active ChatGPT account model metadata") + return _EMPTY_METADATA diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 101be58d197..ee3ae2c45c1 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -1,6 +1,6 @@ import hashlib import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Annotated, Final, TypeAlias @@ -10,12 +10,15 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.utils import _add_path_to_api_base # pyright: ignore[reportPrivateUsage] # shared provider URL helper MODEL_INFO_REFRESH_SECONDS: Final = 300 MODEL_INFO_REFRESH_CONCURRENCY: Final = 8 -MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset({"hosted_vllm", "openai", "text-completion-openai", "openai_like"}) -_EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({}) +MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset( + {"hosted_vllm", "openai", "text-completion-openai", "openai_like", "openrouter", "vercel_ai_gateway"} +) +_EMPTY_LIMITS: Final[Mapping[str, object]] = MappingProxyType({}) def _positive_limit(value: object) -> int | None: @@ -25,30 +28,105 @@ def _positive_limit(value: object) -> int | None: _TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] -class _ModelCard(BaseModel): - model_config = ConfigDict(frozen=True) +class _Modalities(BaseModel): + input: Sequence[str] | None = None + output: Sequence[str] | None = None + +class _Architecture(BaseModel): + input_modalities: Sequence[str] | None = None + output_modalities: Sequence[str] | None = None + + +class _TopProvider(BaseModel): + context_length: _TokenLimit = None + max_completion_tokens: _TokenLimit = None + + +class _Reasoning(BaseModel): + supported_efforts: Sequence[str] | None = None + default_effort: str | None = None + + +class _ReasoningOption(BaseModel): + type: str + values: Sequence[str] | None = None + + +class _ModelCard(GatewayModelMetadata): id: str max_model_len: _TokenLimit = None context_length: _TokenLimit = None - max_input_tokens: _TokenLimit = None - max_output_tokens: _TokenLimit = None + max_tokens: _TokenLimit = None + supported_parameters: Sequence[str] | None = None + modalities: _Modalities | None = None + architecture: _Architecture | None = None + top_provider: _TopProvider | None = None + reasoning: _Reasoning | None = None + reasoning_options: tuple[_ReasoningOption, ...] = () - def token_limits(self) -> Mapping[str, int]: - context: Final = self.max_model_len or self.context_length - input_limit: Final = self.max_input_tokens or context - output_limit: Final = self.max_output_tokens or context - return MappingProxyType( - { - key: value - for key, value in ( - ("max_tokens", context), - ("max_input_tokens", min(input_limit, context) if input_limit and context else input_limit), - ("max_output_tokens", min(output_limit, context) if output_limit and context else output_limit), - ) - if value is not None - } + def token_limits(self, provider: str) -> Mapping[str, object]: + context: Final = self.context_window or self.max_model_len or self.context_length + parameters: Final = self.supported_parameters + supported_tools: Final = "tools" in parameters if parameters is not None else None + supported_reasoning: Final = ( + "reasoning" in parameters or "reasoning_effort" in parameters if parameters is not None else None ) + metadata: Final = GatewayModelMetadata( + context_window=( + self.top_provider.context_length or context + if provider == "openrouter" and self.top_provider is not None + else context + ), + max_input_tokens=self.max_input_tokens, + max_output_tokens=( + self.max_output_tokens + or (self.max_tokens if provider == "vercel_ai_gateway" else None) + or ( + self.top_provider.max_completion_tokens + if provider == "openrouter" and self.top_provider is not None + else None + ) + ), + supports_function_calling=self.supports_function_calling + if self.supports_function_calling is not None + else supported_tools, + supports_parallel_function_calling=self.supports_parallel_function_calling, + supports_reasoning=self.supports_reasoning if self.supports_reasoning is not None else supported_reasoning, + reasoning_effort_levels=( + self.reasoning_effort_levels + if self.reasoning_effort_levels is not None + else ( + self.reasoning.supported_efforts + if provider == "openrouter" and self.reasoning is not None + else next((option.values for option in self.reasoning_options if option.type == "effort"), None) + if provider == "vercel_ai_gateway" + else None + ) + ), + default_reasoning_effort=self.default_reasoning_effort + or (self.reasoning.default_effort if provider == "openrouter" and self.reasoning is not None else None), + supported_endpoints=self.supported_endpoints, + supported_modalities=self.supported_modalities + if self.supported_modalities is not None + else ( + self.architecture.input_modalities + if provider == "openrouter" and self.architecture is not None + else self.modalities.input + if provider == "vercel_ai_gateway" and self.modalities is not None + else None + ), + supported_output_modalities=self.supported_output_modalities + if self.supported_output_modalities is not None + else ( + self.architecture.output_modalities + if provider == "openrouter" and self.architecture is not None + else self.modalities.output + if provider == "vercel_ai_gateway" and self.modalities is not None + else None + ), + ) + return MappingProxyType(metadata.model_dump(mode="json", exclude_none=True)) class _ModelList(BaseModel): @@ -61,17 +139,18 @@ async def get_openai_compatible_model_info( *, model: str, api_base: str, + provider: str = "openai", headers: Mapping[str, str], client: AsyncHTTPHandler, cache: InMemoryCache, -) -> Mapping[str, int]: +) -> Mapping[str, object]: url: Final = _add_path_to_api_base(api_base, "/v1/models") cache_key: Final = ( "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() ) cached: Final[object] = cache.get_cache(cache_key) if isinstance(cached, _ModelList): - return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS) + return next((card.token_limits(provider) for card in cached.data if card.id == model), _EMPTY_LIMITS) try: response: Final = await client.get( @@ -89,4 +168,4 @@ async def get_openai_compatible_model_info( return _EMPTY_LIMITS cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS) - return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS) + return next((card.token_limits(provider) for card in models.data if card.id == model), _EMPTY_LIMITS) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b993a5d8d6d..cf7536182a5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -58,6 +58,7 @@ from litellm.types.proxy.carried_budget_state import ( UserBudgetSnapshot, ) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry +from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings from litellm.types.router import RouterErrors, UpdateRouterConfig from litellm.types.router_weights import validate_router_settings_dict @@ -1208,7 +1209,7 @@ class ModelInfoDelete(LiteLLMPydanticObjectBase): id: str -class ModelInfo(LiteLLMPydanticObjectBase): +class ModelInfo(GatewayModelMetadata, LiteLLMPydanticObjectBase): id: str | None mode: Literal["embedding", "chat", "completion"] | None input_cost_per_token: float | None = 0.0 @@ -1225,7 +1226,7 @@ class ModelInfo(LiteLLMPydanticObjectBase): ) discoverable: bool | None = None - model_config = ConfigDict(protected_namespaces=(), extra="allow") + model_config = ConfigDict(protected_namespaces=(), extra="allow", frozen=False) @model_validator(mode="before") @classmethod diff --git a/litellm/proxy/model_discovery.py b/litellm/proxy/model_discovery.py new file mode 100644 index 00000000000..3430121353c --- /dev/null +++ b/litellm/proxy/model_discovery.py @@ -0,0 +1,59 @@ +from collections.abc import Sequence +from typing import Final, Literal + +from pydantic import TypeAdapter + +from litellm.types.proxy.model_listing import ModelDiscoveryInfo +from litellm.types.proxy.model_metadata import GatewayModelMetadata + +_ADAPTER: Final = TypeAdapter(ModelDiscoveryInfo) + + +def guaranteed_token_limit( + sources: tuple[GatewayModelMetadata, ...], field: Literal["context_window", "max_input_tokens", "max_output_tokens"] +) -> int | None: + values: Final = tuple( + source.context_window + if field == "context_window" + else source.max_input_tokens + if field == "max_input_tokens" + else source.max_output_tokens + for source in sources + ) + if not values or any(value is None for value in values): + return None + return min(value for value in values if value is not None) + + +def _common_boolean(values: tuple[bool | None, ...]) -> bool | None: + return all(values) if values and all(value is not None for value in values) else None + + +def _common_list(values: tuple[Sequence[str] | None, ...]) -> Sequence[str] | None: + if not values or any(value is None for value in values): + return None + first: Final = values[0] + if first is None: + return None + return tuple(item for item in first if all(value is not None and item in value for value in values)) + + +def discover_model_metadata(sources: tuple[GatewayModelMetadata, ...]) -> ModelDiscoveryInfo: + efforts: Final = _common_list(tuple(source.reasoning_effort_levels for source in sources)) + defaults: Final = tuple(source.default_reasoning_effort for source in sources) + default: Final = defaults[0] if defaults and all(value == defaults[0] for value in defaults) else None + valid_default: Final = default if efforts is None or default in efforts else None + metadata: Final = GatewayModelMetadata( + context_window=guaranteed_token_limit(sources, "context_window"), + supports_function_calling=_common_boolean(tuple(source.supports_function_calling for source in sources)), + supports_parallel_function_calling=_common_boolean( + tuple(source.supports_parallel_function_calling for source in sources) + ), + supports_reasoning=_common_boolean(tuple(source.supports_reasoning for source in sources)), + reasoning_effort_levels=efforts, + default_reasoning_effort=valid_default, + supported_endpoints=_common_list(tuple(source.supported_endpoints for source in sources)), + supported_modalities=_common_list(tuple(source.supported_modalities for source in sources)), + supported_output_modalities=_common_list(tuple(source.supported_output_modalities for source in sources)), + ) + return _ADAPTER.validate_python(metadata.model_dump(mode="json", exclude_none=True)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ed2a5b89a32..9105c3078a9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -136,6 +136,7 @@ from litellm.proxy.common_utils.callback_utils import ( ) from litellm.proxy.common_utils.realtime_utils import _realtime_request_body from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog +from litellm.proxy.model_discovery import discover_model_metadata from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, @@ -158,6 +159,7 @@ from litellm.router_utils.auto_router_tuning_baseline import ( from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.router_utils.routing_groups import parse_routing_groups from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.types.proxy.model_metadata import GatewayModelMetadata, resolve_gateway_model_metadata from litellm.types.utils import ( PRICING_OVERRIDES_KEY, ModelResponse, @@ -1592,12 +1594,12 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState None if model_info_refresh_disabled else scheduler if scheduler is not None else AsyncIOScheduler() ) if model_info_scheduler is not None: + await ProxyStartupEvent.refresh_model_info() model_info_scheduler.add_job( ProxyStartupEvent.refresh_model_info, "interval", seconds=MODEL_INFO_REFRESH_SECONDS, id="refresh_model_info", - next_run_time=datetime.now(timezone.utc), max_instances=1, replace_existing=True, ) @@ -14703,10 +14705,22 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } - # don't return the api key / vertex credentials - # don't return the llm credentials - model = remove_sensitive_info_from_deployment(model, excluded_keys={"litellm_credential_name"}) - return model + gateway_metadata: Final = resolve_gateway_model_metadata( + MappingProxyType({**litellm_model_info, **discovered_model_info}), model_info + ) + published_model: Final = { + **model, + "model_info": { + **{ + field: value + for field, value in model["model_info"].items() + if field not in GatewayModelMetadata.model_fields + }, + **gateway_metadata.model_dump(mode="json", exclude_none=True), + **discover_model_metadata((gateway_metadata,)), + }, + } + return remove_sensitive_info_from_deployment(published_model, excluded_keys={"litellm_credential_name"}) async def _get_caller_byok_team_scope( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 29f2f46f001..26c410e4827 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -27,7 +27,7 @@ from datetime import date, datetime, timedelta, timezone from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from functools import partial -from itertools import takewhile +from itertools import chain, takewhile from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -78,9 +78,11 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, with_litellm_call_id, ) +from litellm.proxy.model_discovery import discover_model_metadata, guaranteed_token_limit from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse +from litellm.types.proxy.model_metadata import resolve_gateway_model_metadata from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage try: @@ -8756,6 +8758,9 @@ async def get_available_models_for_user( return capped +_MODEL_LISTING_RESPONSE_ADAPTER: Final = TypeAdapter(ModelInfoResponse) + + def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None: try: return get_model_info(model) @@ -8838,6 +8843,15 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: return max(limits) if limits else None +def _merged_listing_info(candidates: tuple[ModelInfo, ...]) -> Mapping[str, object]: + fields: Final = tuple(dict.fromkeys(chain.from_iterable(candidate.keys() for candidate in candidates))) + return { + field: next(candidate.get(field) for candidate in candidates if candidate.get(field) is not None) + for field in fields + if any(candidate.get(field) is not None for candidate in candidates) + } + + def create_model_info_response( model_id: str, provider: str, @@ -8918,8 +8932,40 @@ def create_model_info_response( if max_output_tokens is not None: base["max_output_tokens"] = max_output_tokens + candidates_by_model: Final = dict(zip(deployment_models, candidate_sets)) + discovery_sources: Final = ( + tuple( + resolve_gateway_model_metadata( + _merged_listing_info(candidates_by_model.get(deployment.cost_map_key, candidate_sets[0])), + deployment.model_info, + ) + for deployment in listing_info.deployments + ) + if listing_info is not None and listing_info.deployments + else tuple( + resolve_gateway_model_metadata(_merged_listing_info(candidates), {}) for candidates in candidate_sets + ) + ) + has_context_contract: Final = any(source.context_window is not None for source in discovery_sources) + limits: Final = ( + { + field: value + for field in ("max_input_tokens", "max_output_tokens") + if (value := guaranteed_token_limit(discovery_sources, field)) is not None + } + if has_context_contract + else {field: value for field, value in base.items() if field in ("max_input_tokens", "max_output_tokens")} + ) + discovered_base: Final[ModelInfoResponse] = _MODEL_LISTING_RESPONSE_ADAPTER.validate_python( + { + **{field: value for field, value in base.items() if field not in ("max_input_tokens", "max_output_tokens")}, + **limits, + **discover_model_metadata(discovery_sources), + }, + ) + if not include_metadata: - return base + return discovered_base effective_fallback_type: Final = fallback_type if fallback_type is not None else "general" @@ -8935,7 +8981,7 @@ def create_model_info_response( llm_router=llm_router, fallback_type=effective_fallback_type, ) - return {**base, "metadata": {"fallbacks": fallbacks}} + return {**discovered_base, "metadata": {"fallbacks": fallbacks}} def validate_model_access( diff --git a/litellm/router.py b/litellm/router.py index 0b9f12c8da3..f783bcb9cfe 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -292,6 +292,7 @@ from litellm.types.router import ( LiteLLM_Params, MockRouterTestingParams, ModelGroupInfo, + ModelListingDeployment, OptionalPreCallChecks, PreRoutingStrategy, RetryPolicy, @@ -10395,30 +10396,7 @@ class Router: } ) ) - model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) - if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: - return - if api_base is None or "*" in model or params.get("use_clientside_credentials"): - return - api_key: Final = params.api_key or dynamic_api_key - headers: Final = TypeAdapter(Mapping[str, str]).validate_python( - params.get("extra_headers") or params.get("headers") or MappingProxyType({}) - ) - auth_headers: Final = ( - MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) - ) - limits: Final = await get_openai_compatible_model_info( - model=model, - api_base=api_base, - headers=MappingProxyType( - { - **auth_headers, - **MappingProxyType({key.lower(): value for key, value in headers.items()}), - } - ), - client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), - cache=self.cache.in_memory_cache, - ) + limits: Final = await self._aget_deployment_model_metadata(params, client=client) model_id: Final = deployment.model_info.id if not limits or model_id is None or self.get_model_info(model_id) is not raw_deployment: return @@ -10429,7 +10407,51 @@ class Router: ) self._invalidate_model_group_info_cache() - def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + async def _aget_deployment_model_metadata( + self, params: LiteLLM_Params, *, client: AsyncHTTPHandler | None + ) -> Mapping[str, object]: + if params.get("use_clientside_credentials") or "*" in params.model: + return MappingProxyType({}) + if params.model.startswith("chatgpt/"): + from litellm.llms.chatgpt.model_info import get_chatgpt_model_info + + return await get_chatgpt_model_info( + model=params.model.removeprefix("chatgpt/"), + api_base=params.api_base, + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self.cache.in_memory_cache, + ) + model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) + if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return MappingProxyType({}) + discovery_api_base: Final = api_base or { + "openrouter": "https://openrouter.ai/api/v1", + "vercel_ai_gateway": "https://ai-gateway.vercel.sh/v1", + }.get(provider) + if discovery_api_base is None or "*" in model or params.get("use_clientside_credentials"): + return MappingProxyType({}) + api_key: Final = params.api_key or dynamic_api_key + headers: Final = TypeAdapter(Mapping[str, str]).validate_python( + params.get("extra_headers") or params.get("headers") or MappingProxyType({}) + ) + auth_headers: Final = ( + MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) + ) + return await get_openai_compatible_model_info( + model=model, + api_base=discovery_api_base, + provider=provider, + headers=MappingProxyType( + { + **auth_headers, + **MappingProxyType({key.lower(): value for key, value in headers.items()}), + } + ), + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self.cache.in_memory_cache, + ) + + def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, object]: cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) if ( model_id is not None @@ -10496,6 +10518,15 @@ class Router: cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + deployments=tuple( + ModelListingDeployment( + cost_map_key=( + model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") + ), + model_info=model_info, + ) + for model_info, litellm_params in zip(model_infos, params) + ), ) @staticmethod diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index 24cfa85eee4..8242ca991c5 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -1,15 +1,29 @@ +from collections.abc import Sequence + """Response types for the model listing/retrieve endpoints (/v1/models, /models).""" from typing import Literal -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict + + +class ModelDiscoveryInfo(TypedDict, total=False): + context_window: ReadOnly[int] + supports_function_calling: ReadOnly[bool] + supports_parallel_function_calling: ReadOnly[bool] + supports_reasoning: ReadOnly[bool] + reasoning_effort_levels: ReadOnly[Sequence[str]] + default_reasoning_effort: ReadOnly[str] + supported_endpoints: ReadOnly[Sequence[str]] + supported_modalities: ReadOnly[Sequence[str]] + supported_output_modalities: ReadOnly[Sequence[str]] class ModelInfoMetadata(TypedDict): fallbacks: list[str] -class ModelInfoResponse(TypedDict): +class ModelInfoResponse(ModelDiscoveryInfo): """OpenAI-compatible model object. `mode`, `max_input_tokens`, and `max_output_tokens` are attached when the cost map or deployment config knows them; `metadata` is present only with include_metadata=true. diff --git a/litellm/types/proxy/model_metadata.py b/litellm/types/proxy/model_metadata.py new file mode 100644 index 00000000000..0404bc770c1 --- /dev/null +++ b/litellm/types/proxy/model_metadata.py @@ -0,0 +1,85 @@ +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, ValidationInfo, field_validator + +_STRING_VALUES_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_VALIDATED_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +class GatewayModelMetadata(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + context_window: int | None = Field( + default=None, gt=0, description="Maximum combined input and generated output tokens" + ) + max_input_tokens: int | None = Field(default=None, gt=0, description="Independent maximum input tokens") + max_output_tokens: int | None = Field(default=None, gt=0, description="Independent maximum generated output tokens") + supports_function_calling: bool | None = None + supports_parallel_function_calling: bool | None = None + supports_reasoning: bool | None = None + reasoning_effort_levels: Sequence[str] | None = Field( + default=None, description="Exact effort values this route accepts" + ) + default_reasoning_effort: str | None = Field(default=None, description="Effort used when the request omits it") + supported_endpoints: Sequence[str] | None = Field( + default=None, description="Inference endpoints exposed by this route" + ) + supported_modalities: Sequence[str] | None = None + supported_output_modalities: Sequence[str] | None = None + + @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") + @classmethod + def positive_limit(cls, value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + + @field_validator( + "supports_function_calling", "supports_parallel_function_calling", "supports_reasoning", mode="before" + ) + @classmethod + def known_boolean(cls, value: object) -> bool | None: + return value if isinstance(value, bool) else None + + @field_validator( + "reasoning_effort_levels", + "supported_endpoints", + "supported_modalities", + "supported_output_modalities", + mode="before", + ) + @classmethod + def known_string_list(cls, value: object) -> Sequence[str] | None: + try: + return tuple(dict.fromkeys(_STRING_VALUES_ADAPTER.validate_python(value))) + except ValidationError: + return None + + @field_validator("default_reasoning_effort", mode="before") + @classmethod + def known_string(cls, value: object) -> str | None: + return value if isinstance(value, str) else None + + @field_validator("reasoning_effort_levels") + @classmethod + def enabled_reasoning_levels(cls, value: Sequence[str] | None, info: ValidationInfo) -> Sequence[str] | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) + return None if data.get("supports_reasoning") is False else value + + @field_validator("default_reasoning_effort") + @classmethod + def supported_default(cls, value: str | None, info: ValidationInfo) -> str | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) + levels: Final = data.get("reasoning_effort_levels") + if data.get("supports_reasoning") is False: + return None + return value if levels is None or isinstance(levels, (list, tuple)) and value in levels else None + + +def resolve_gateway_model_metadata( + catalog: Mapping[str, object], configured: Mapping[str, object] +) -> GatewayModelMetadata: + values: Final = { + field: configured[field] if configured.get(field) is not None else catalog.get(field) + for field in GatewayModelMetadata.model_fields + } + return GatewayModelMetadata.model_validate(values) diff --git a/litellm/types/router.py b/litellm/types/router.py index e131a7184d4..efedacdc766 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -772,7 +772,13 @@ class Deployment(BaseModel): @dataclass(frozen=True, slots=True) class DiscoveredDeploymentModelInfo: deployment: Mapping[str, object] - limits: Mapping[str, int] + limits: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class ModelListingDeployment: + cost_map_key: str | None + model_info: Mapping[str, object] @dataclass(frozen=True, slots=True) @@ -791,6 +797,7 @@ class DeploymentModelListingInfo: cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None + deployments: tuple[ModelListingDeployment, ...] = () class RouterErrors(enum.Enum): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 142f9b14a72..dca4be78081 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -207,7 +207,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_xhigh_reasoning_effort: bool | None supports_max_reasoning_effort: bool | None reasoning_effort_levels: ReadOnly[Sequence[str] | None] - default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None] + default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None] supports_output_config: bool | None supports_image_size: bool | None supports_anthropic_thinking_payload: ReadOnly[bool | None] @@ -275,6 +275,9 @@ class OffPeakPricing(TypedDict, total=False): class ModelInfoBase(ProviderSpecificModelInfo, total=False): + context_window: ReadOnly[int | None] + supported_modalities: ReadOnly[Sequence[str] | None] + supported_output_modalities: ReadOnly[Sequence[str] | None] key: Required[str] # the key in litellm.model_cost which is returned max_tokens: Required[int | None] diff --git a/litellm/utils.py b/litellm/utils.py index a7d7447d7f3..4a6adbd379d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6092,6 +6092,9 @@ def _get_model_info_helper( returned_model_info: Final = ModelInfoBase( key=key, + context_window=_model_info.get("context_window", None), + supported_modalities=_model_info.get("supported_modalities", None), + supported_output_modalities=_model_info.get("supported_output_modalities", None), max_tokens=_model_info.get("max_tokens", None), max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index cc20a6ff544..68be5bd1971 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -267,6 +267,11 @@ "comment": { "type": "string" }, + "context_window": { + "type": "integer", + "minimum": 1, + "description": "Maximum combined input and generated output tokens, independent of the input and output limits." + }, "cost_per_second": { "type": "number", "minimum": 0 @@ -280,7 +285,8 @@ "low", "medium", "high", - "xhigh" + "xhigh", + "max" ] }, "deprecation_date": { @@ -502,7 +508,7 @@ "max_input_tokens": { "type": "integer", "minimum": 0, - "description": "Maximum prompt/context tokens the model accepts." + "description": "Maximum input tokens, independent of the combined input and output context window." }, "max_output_tokens": { "type": "integer", diff --git a/tests/unit/llms/chatgpt/test_model_info.py b/tests/unit/llms/chatgpt/test_model_info.py new file mode 100644 index 00000000000..98e2c818af3 --- /dev/null +++ b/tests/unit/llms/chatgpt/test_model_info.py @@ -0,0 +1,168 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.chatgpt.authenticator import Authenticator +from litellm.llms.chatgpt.common_utils import GetAccessTokenError +from litellm.llms.chatgpt.model_info import _ChatGPTModel, get_chatgpt_model_info +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + +async def test_account_catalog_preserves_client_policy_separately_from_model_limits( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text( + json.dumps({"access_token": "local-token", "account_id": "local-account", "expires_at": 10**30}) + ) + card: Final = { + "slug": "fixture-model", + "context_window": 600, + "max_context_window": 1200, + "supported_reasoning_levels": [{"effort": "high"}, {"effort": "future-effort"}], + "default_reasoning_level": "future-effort", + "input_modalities": ["text", "image"], + "supports_parallel_tool_calls": False, + } + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/backend-api/codex/models" + assert request.url.params.get("client_version") + assert request.headers["authorization"] == "Bearer local-token" + assert request.headers["chatgpt-account-id"] == "local-account" + assert request.headers["accept"] == "application/json" + return httpx.Response(200, json={"models": [card]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + cache: Final = InMemoryCache() + result: Final = await get_chatgpt_model_info( + model="fixture-model", + api_base="https://account.test/backend-api/codex/", + authenticator=Authenticator(), + client=handler, + cache=cache, + ) + assert result == { + "codex_context_window": card["context_window"], + "codex_max_context_window": card["max_context_window"], + "supports_parallel_function_calling": card["supports_parallel_tool_calls"], + "supports_reasoning": True, + "reasoning_effort_levels": tuple(level["effort"] for level in card["supported_reasoning_levels"]), + "default_reasoning_effort": card["default_reasoning_level"], + "supported_endpoints": ("/responses",), + "supported_modalities": tuple(card["input_modalities"]), + } + assert ( + await get_chatgpt_model_info( + model="not-selected", + api_base="https://account.test/backend-api/codex/", + authenticator=Authenticator(), + client=handler, + cache=cache, + ) + == {} + ) + + +async def test_switching_accounts_invalidates_catalog_without_exposing_credentials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + auth_file: Final = tmp_path / "auth.json" + cache: Final = InMemoryCache() + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + + def respond(request: httpx.Request) -> httpx.Response: + capacity: Final = 1024 if request.headers["chatgpt-account-id"] == "first" else 2048 + return httpx.Response(200, json={"models": [{"slug": "fixture", "context_window": capacity}]}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + for account, capacity in (("first", 1024), ("second", 2048)): + auth_file.write_text( + json.dumps({"access_token": "local-token", "account_id": account, "expires_at": 10**30}) + ) + result: Final = await get_chatgpt_model_info( + model="fixture", + api_base="https://account.test/backend-api/codex", + authenticator=Authenticator(), + client=handler, + cache=cache, + ) + assert result["codex_context_window"] == capacity + assert "local-token" not in json.dumps(dict(result)) + + +async def test_discovery_without_authorization_never_starts_device_login( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + with pytest.raises(GetAccessTokenError, match="model discovery cannot start device login"): + Authenticator().get_access_token(allow_device_login=False) + + def reject(request: httpx.Request) -> httpx.Response: + pytest.fail("Discovery without credentials must not send any request") + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(reject)) as client: + handler.client = client + assert ( + await get_chatgpt_model_info( + model="fixture", + authenticator=Authenticator(), + client=handler, + cache=InMemoryCache(), + ) + == {} + ) + assert not (tmp_path / "auth.json").exists() + + +@pytest.mark.parametrize(("status", "body"), ((503, {}), (200, {"models": "invalid"}))) +async def test_invalid_or_unavailable_catalog_preserves_fallback( + status: int, body: dict[str, object], tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_text(json.dumps({"access_token": "local", "expires_at": 10**30})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda _: httpx.Response(status, json=body))) as client: + handler.client = client + assert ( + await get_chatgpt_model_info( + model="fixture", + authenticator=Authenticator(), + client=handler, + cache=InMemoryCache(), + ) + == {} + ) + + +@pytest.mark.parametrize( + ("default", "multi_agent", "expected"), + (("ultra", "high", "high"), ("ultra", None, "max"), ("persistent", None, "disabled")), +) +def test_codex_ui_efforts_are_not_advertised_as_api_efforts( + default: str, multi_agent: str | None, expected: str +) -> None: + metadata: Final = _ChatGPTModel.model_validate( + { + "slug": "fixture", + "supported_reasoning_levels": [{"effort": effort} for effort in ("high", "max", "ultra", "persistent")], + "default_reasoning_level": default, + "multi_agent_reasoning_effort": multi_agent, + } + ).metadata() + assert metadata["reasoning_effort_levels"] == ("high", "max", "disabled") + assert metadata["default_reasoning_effort"] == expected diff --git a/tests/unit/llms/openai_like/test_model_info.py b/tests/unit/llms/openai_like/test_model_info.py index 15a7d9e7fc6..49edd3ea6ec 100644 --- a/tests/unit/llms/openai_like/test_model_info.py +++ b/tests/unit/llms/openai_like/test_model_info.py @@ -16,14 +16,14 @@ from litellm.llms.openai_like.model_info import ( @pytest.mark.parametrize( ("card", "expected"), ( - ({"max_model_len": 8192}, {"max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192}), + ({"max_model_len": 8192}, {"context_window": 8192}), ( {"context_length": 4096, "max_output_tokens": 1024}, - {"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 1024}, + {"context_window": 4096, "max_output_tokens": 1024}, ), ( {"max_model_len": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192}, - {"max_tokens": 4096, "max_input_tokens": 2048, "max_output_tokens": 4096}, + {"context_window": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192}, ), ({"max_input_tokens": 2048}, {"max_input_tokens": 2048}), ({"max_output_tokens": 1024}, {"max_output_tokens": 1024}), @@ -32,7 +32,9 @@ from litellm.llms.openai_like.model_info import ( ({}, {}), ), ) -async def test_discovers_only_valid_advertised_limits(card: Mapping[str, object], expected: Mapping[str, int]) -> None: +async def test_discovers_only_valid_advertised_limits( + card: Mapping[str, object], expected: Mapping[str, object] +) -> None: def respond(request: httpx.Request) -> httpx.Response: assert request.url.path == "/tenant/v1/models" assert request.headers["authorization"] == "Bearer local-key" @@ -81,18 +83,18 @@ async def test_cache_is_scoped_to_endpoint_and_authentication_and_expires() -> N handler.client = client cache: Final = InMemoryCache(clock=clock) - async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, int]: + async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, object]: return await get_openai_compatible_model_info( model=model, api_base=f"https://{host}", headers={"Authorization": key}, client=handler, cache=cache ) - assert (await lookup())["max_input_tokens"] == 1024 - assert (await lookup("second"))["max_input_tokens"] == 2048 + assert (await lookup())["context_window"] == 1024 + assert (await lookup("second"))["context_window"] == 2048 assert responder.call_count == 1 - assert (await lookup(key="two"))["max_input_tokens"] == 4096 - assert (await lookup(host="two.test"))["max_input_tokens"] == 8192 + assert (await lookup(key="two"))["context_window"] == 4096 + assert (await lookup(host="two.test"))["context_window"] == 8192 clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1 - assert (await lookup())["max_input_tokens"] == 16384 + assert (await lookup())["context_window"] == 16384 assert responder.call_count == 4 @@ -124,3 +126,70 @@ async def test_unavailable_metadata_is_best_effort_and_negative_cached( == {} ) assert responder.call_count == 1 + + +@pytest.mark.parametrize( + ("provider", "card", "expected"), + ( + ( + "vercel_ai_gateway", + { + "context_window": 1200, + "max_tokens": 200, + "modalities": {"input": ["text", "image", "pdf"], "output": ["text"]}, + "supported_parameters": ["tools", "reasoning"], + "reasoning_options": [{"type": "effort", "values": ["low", "xhigh", "max"]}], + }, + { + "context_window": 1200, + "max_output_tokens": 200, + "supported_modalities": ["text", "image", "pdf"], + "supported_output_modalities": ["text"], + "supports_function_calling": True, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "xhigh", "max"], + }, + ), + ( + "openrouter", + { + "context_length": 1200, + "top_provider": {"context_length": 1100, "max_completion_tokens": 200}, + "architecture": {"input_modalities": ["text", "file"], "output_modalities": ["text"]}, + "supported_parameters": ["tools", "reasoning_effort"], + "reasoning": {"supported_efforts": ["low", "max"], "default_effort": "low"}, + }, + { + "context_window": 1100, + "max_output_tokens": 200, + "supported_modalities": ["text", "file"], + "supported_output_modalities": ["text"], + "supports_function_calling": True, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + }, + ), + ( + "openai", + {"max_tokens": 200, "supported_parameters": []}, + {"supports_function_calling": False, "supports_reasoning": False}, + ), + ), +) +async def test_normalizes_supplier_metadata_without_inventing_input_limit(provider, card, expected): + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [{"id": "selected", **card}]})) + ) as client: + handler.client = client + result: Final = await get_openai_compatible_model_info( + model="selected", + provider=provider, + api_base="https://supplier.test/v1", + headers={}, + client=handler, + cache=InMemoryCache(), + ) + assert result == expected diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 5175d92084c..5f5a3099f62 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -85,6 +85,7 @@ async def test_discovery_preserves_model_info_fallbacks( { "id": backend_model.split("/", 1)[1], "max_model_len": advertised_limit, + "max_input_tokens": advertised_limit, } ] }, @@ -142,7 +143,9 @@ async def test_upstream_limits_reach_model_info_routes( def respond(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/models" - return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": 4096}]}) + return httpx.Response( + 200, json={"data": [{"id": "org/local-model", "max_model_len": 4096, "max_input_tokens": 4096}]} + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() diff --git a/tests/unit/proxy/test_model_discovery.py b/tests/unit/proxy/test_model_discovery.py new file mode 100644 index 00000000000..c56ec401051 --- /dev/null +++ b/tests/unit/proxy/test_model_discovery.py @@ -0,0 +1,67 @@ +from typing import Final + +import pytest + +from litellm.proxy.model_discovery import discover_model_metadata, guaranteed_token_limit +from litellm.types.proxy.model_metadata import GatewayModelMetadata, resolve_gateway_model_metadata + + +def test_configured_overrides_preserve_false_empty_and_independent_limits(): + metadata: Final = resolve_gateway_model_metadata( + { + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "supports_function_calling": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + }, + {"context_window": 1000, "supports_function_calling": False, "reasoning_effort_levels": []}, + ) + assert metadata.context_window == 1000 + assert metadata.max_input_tokens == 1100 + assert metadata.max_output_tokens == 200 + assert discover_model_metadata((metadata,)) == { + "context_window": 1000, + "supports_function_calling": False, + "reasoning_effort_levels": [], + } + + +def test_mixed_deployments_advertise_only_guaranteed_metadata(): + sources: Final = ( + GatewayModelMetadata( + context_window=1200, + max_input_tokens=1100, + max_output_tokens=200, + supports_function_calling=True, + supports_parallel_function_calling=True, + reasoning_effort_levels=["low", "high", "max"], + default_reasoning_effort="high", + supported_modalities=["text", "image"], + ), + GatewayModelMetadata( + context_window=800, + max_input_tokens=700, + max_output_tokens=100, + supports_function_calling=False, + reasoning_effort_levels=["low", "high"], + default_reasoning_effort="low", + supported_modalities=["text"], + ), + ) + assert discover_model_metadata(sources) == { + "context_window": 800, + "supports_function_calling": False, + "reasoning_effort_levels": ["low", "high"], + "supported_modalities": ["text"], + } + assert guaranteed_token_limit(sources, "max_input_tokens") == 700 + assert guaranteed_token_limit(sources, "max_output_tokens") == 100 + + +@pytest.mark.parametrize("value", [None, True, 0, -1, "1200", 1.5]) +def test_unknown_or_invalid_context_never_fabricates_capacity(value): + metadata: Final = GatewayModelMetadata(context_window=value, max_input_tokens=1100, max_output_tokens=200) + assert discover_model_metadata((metadata,)) == {} + assert discover_model_metadata((GatewayModelMetadata(context_window=1200), metadata)) == {} diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 0a095183b6e..8b868db91f9 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2433,3 +2433,66 @@ def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog) assert provider_result.code == internal_result.code == "500" assert ISSUE_URL_BASE in caplog.text assert ISSUE_URL_BASE not in internal_result.message + + +def test_model_listing_advertises_complete_configured_metadata_and_no_secrets(): + metadata: Final = { + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "supports_function_calling": True, + "supports_parallel_function_calling": False, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "xhigh", "max"], + "default_reasoning_effort": "xhigh", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + } + router: Final = litellm.Router( + model_list=[ + { + "model_name": "selected", + "litellm_params": {"model": "openai/unknown-test-model", "api_key": "private-fixture-key"}, + "model_info": {**metadata, "internal_secret": "must-not-advertise"}, + } + ] + ) + response: Final = create_model_info_response( + "selected", "openai", llm_router=router, get_model_info=_raise_unmapped + ) + assert response == { + "id": "selected", + "object": "model", + "created": response["created"], + "owned_by": "openai", + **metadata, + } + + +def test_model_listing_context_contract_uses_minimum_and_omits_unknown_group_limits(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "selected", + "litellm_params": {"model": "openai/unknown-one"}, + "model_info": {"context_window": 1200, "max_input_tokens": 1100, "max_output_tokens": 200}, + }, + { + "model_name": "selected", + "litellm_params": {"model": "openai/unknown-two"}, + "model_info": {"context_window": 800, "max_output_tokens": 100}, + }, + ] + ) + response: Final = create_model_info_response( + "selected", "openai", llm_router=router, get_model_info=_raise_unmapped + ) + assert response == { + "id": "selected", + "object": "model", + "created": response["created"], + "owned_by": "openai", + "context_window": 800, + "max_output_tokens": 100, + } diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index a05c345b5cd..95a8e6cfb93 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -533,3 +533,29 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): "unknown_root-new_family_models", "vertex_ai-new_family_models", ] + + +@pytest.mark.parametrize("context", [0, -1, True, 1.5, "1200"]) +def test_schema_rejects_invalid_total_context(committed_schema, context): + assert not build_validator(committed_schema).is_valid( + {"fixture-model": {"litellm_provider": "openai", "context_window": context}} + ) + + +def test_schema_and_model_info_preserve_independent_total_context(committed_schema, monkeypatch): + entry = { + "litellm_provider": "openai", + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "max_tokens": 200, + } + assert build_validator(committed_schema).is_valid({"fixture-model": entry}) + monkeypatch.setattr(litellm, "model_cost", {"fixture-model": entry}) + info = litellm.get_model_info("fixture-model") + assert (info["context_window"], info["max_input_tokens"], info["max_output_tokens"], info["max_tokens"]) == ( + 1200, + 1100, + 200, + 200, + ) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index ff8c91cae70..3f2bf57d2c1 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -87,7 +87,19 @@ async def test_discovered_limits_survive_deployment_growth_and_removal( await handler.client.aclose() async with httpx.AsyncClient( transport=httpx.MockTransport( - lambda request: httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + lambda request: httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + } + ] + }, + ) ) ) as client: handler.client = client @@ -113,30 +125,55 @@ async def test_discovered_limits_survive_deployment_growth_and_removal( async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) - router: Final = Router(model_list=[{ - "model_name": "local", - "litellm_params": { - "model": "hosted_vllm/local-model", - "api_base": "https://original.test/v1", - "api_key": "local-key", - }, - "model_info": {"id": "replaced-deployment"}, - }]) + router: Final = Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://original.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "replaced-deployment"}, + } + ] + ) def respond(request: httpx.Request) -> httpx.Response: if request.url.host == "original.test": - router.upsert_deployment(Deployment( - model_name="local", - litellm_params=LiteLLM_Params( - model="hosted_vllm/local-model", - api_base="https://replacement.test/v1", - api_key="local-key", - ), - model_info=ModelInfo(id="replaced-deployment"), - )) - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}) + router.upsert_deployment( + Deployment( + model_name="local", + litellm_params=LiteLLM_Params( + model="hosted_vllm/local-model", + api_base="https://replacement.test/v1", + api_key="local-key", + ), + model_info=ModelInfo(id="replaced-deployment"), + ) + ) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + } + ] + }, + ) assert request.url.host == "replacement.test" - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]}) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "local-model", "max_model_len": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048} + ] + }, + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -152,15 +189,19 @@ async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) first, second = tuple( - Router(model_list=[{ - "model_name": "local", - "litellm_params": { - "model": "hosted_vllm/local-model", - "api_base": f"https://{host}.test/v1", - "api_key": "local-key", - }, - "model_info": {"id": "shared-discovery-id"}, - }]) + Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": f"https://{host}.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "shared-discovery-id"}, + } + ] + ) for host in ("first", "second") ) @@ -168,7 +209,14 @@ async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: if request.url.host == "unavailable.test": return httpx.Response(503) limit: Final = 8192 if request.url.host == "first.test" else 2048 - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": limit}]}) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "local-model", "max_model_len": limit, "max_input_tokens": limit, "max_output_tokens": limit} + ] + }, + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -181,15 +229,17 @@ async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: assert first.get_configured_token_limits("local") == (8192, 8192) assert second.get_configured_token_limits("local") == (2048, 2048) assert litellm.model_cost["shared-discovery-id"].get("max_input_tokens") is None - first.upsert_deployment(Deployment( - model_name="local", - litellm_params=LiteLLM_Params( - model="hosted_vllm/local-model", - api_base="https://unavailable.test/v1", - api_key="local-key", - ), - model_info=ModelInfo(id="shared-discovery-id"), - )) + first.upsert_deployment( + Deployment( + model_name="local", + litellm_params=LiteLLM_Params( + model="hosted_vllm/local-model", + api_base="https://unavailable.test/v1", + api_key="local-key", + ), + model_info=ModelInfo(id="shared-discovery-id"), + ) + ) assert first.get_configured_token_limits("local") == (None, None) await first.arefresh_model_info(client=handler) assert first.get_configured_token_limits("local") == (None, None) @@ -200,17 +250,19 @@ async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: async def test_discovery_refreshes_other_endpoints_while_one_is_pending(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) second_started: Final = asyncio.Event() - router: Final = Router(model_list=[ - { - "model_name": host, - "litellm_params": { - "model": "hosted_vllm/local-model", - "api_base": f"https://{host}.test/v1", - "api_key": "local-key", - }, - } - for host in ("first", "second", "third") - ]) + router: Final = Router( + model_list=[ + { + "model_name": host, + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": f"https://{host}.test/v1", + "api_key": "local-key", + }, + } + for host in ("first", "second", "third") + ] + ) async def respond(request: httpx.Request) -> httpx.Response: if request.url.host == "first.test": @@ -218,7 +270,14 @@ async def test_discovery_refreshes_other_endpoints_while_one_is_pending(monkeypa if request.url.host == "second.test": second_started.set() return httpx.Response(503) - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]}) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "local-model", "max_model_len": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048} + ] + }, + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -234,21 +293,51 @@ async def test_discovery_refreshes_other_endpoints_while_one_is_pending(monkeypa async def test_discovered_limits_expire_after_the_last_successful_refresh(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) clock: Final = Mock(return_value=0.0) - router: Final = Router(model_list=[{ - "model_name": "local", - "litellm_params": { - "model": "hosted_vllm/local-model", - "api_base": "https://expiry.test/v1", - "api_key": "local-key", - }, - "model_info": {"id": "expiring-discovery"}, - }]) + router: Final = Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://expiry.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "expiring-discovery"}, + } + ] + ) router._discovered_model_info_cache = InMemoryCache(clock=clock, default_ttl=2 * MODEL_INFO_REFRESH_SECONDS) - responses: Final = iter(( - httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}), - httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}), - httpx.Response(503), - )) + responses: Final = iter( + ( + httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + } + ] + }, + ), + httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + } + ] + }, + ), + httpx.Response(503), + ) + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client: @@ -283,7 +372,20 @@ async def test_discovered_limits_are_isolated_overridable_and_refreshable( def respond(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/models" assert request.headers["authorization"] == "Bearer local-key" - return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": next(upstream_limit)}]}) + limit: Final = next(upstream_limit) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/local-model", + "max_model_len": limit, + "max_input_tokens": limit, + "max_output_tokens": limit, + } + ] + }, + ) router: Final = Router( model_list=[ @@ -337,10 +439,24 @@ async def test_discovered_limits_are_isolated_overridable_and_refreshable( async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) - responses: Final = iter(( - httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}), - httpx.Response(503), - )) + responses: Final = iter( + ( + httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + } + ] + }, + ), + httpx.Response(503), + ) + ) def respond(request: httpx.Request) -> httpx.Response: assert request.url.host == "backend.test" @@ -348,27 +464,29 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa assert request.headers["x-tenant"] == "tenant" return next(responses) - router: Final = Router(model_list=[ - { - "model_name": "configured", - "litellm_params": { - "model": "hosted_vllm/local-model", - "api_base": "https://backend.test/v1", - "api_key": "unused-key", - "extra_headers": {"authorization": "Bearer local-key", "X-Tenant": "tenant"}, + router: Final = Router( + model_list=[ + { + "model_name": "configured", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://backend.test/v1", + "api_key": "unused-key", + "extra_headers": {"authorization": "Bearer local-key", "X-Tenant": "tenant"}, + }, + "model_info": {"id": "configured", "max_input_tokens": 1024}, }, - "model_info": {"id": "configured", "max_input_tokens": 1024}, - }, - { - "model_name": "byok", - "litellm_params": { - "model": "openai/local-model", - "api_base": "https://caller.test/v1", - "use_clientside_credentials": True, + { + "model_name": "byok", + "litellm_params": { + "model": "openai/local-model", + "api_base": "https://caller.test/v1", + "use_clientside_credentials": True, + }, }, - }, - {"model_name": "default-openai", "litellm_params": {"model": "openai/local-model", "api_key": "unused"}}, - ]) + {"model_name": "default-openai", "litellm_params": {"model": "openai/local-model", "api_key": "unused"}}, + ] + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() responder: Final = Mock(side_effect=respond) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 34cde5bbef6..c9903caf911 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -50293,6 +50293,16 @@ export interface components { litellm__proxy___types__ModelInfo: { /** Base Model */ base_model: ("gpt-4-1106-preview" | "gpt-4-32k" | "gpt-4" | "gpt-3.5-turbo-16k" | "gpt-3.5-turbo" | "text-embedding-ada-002") | null; + /** + * Context Window + * @description Maximum combined input and generated output tokens + */ + context_window?: number | null; + /** + * Default Reasoning Effort + * @description Effort used when the request omits it + */ + default_reasoning_effort?: string | null; /** Discoverable */ discoverable?: boolean | null; /** Id */ @@ -50302,6 +50312,16 @@ export interface components { * @default 0 */ input_cost_per_token: number | null; + /** + * Max Input Tokens + * @description Independent maximum input tokens + */ + max_input_tokens?: number | null; + /** + * Max Output Tokens + * @description Independent maximum generated output tokens + */ + max_output_tokens?: number | null; /** * Max Tokens * @default 2048 @@ -50314,6 +50334,26 @@ export interface components { * @default 0 */ output_cost_per_token: number | null; + /** + * Reasoning Effort Levels + * @description Exact effort values this route accepts + */ + reasoning_effort_levels?: string[] | null; + /** + * Supported Endpoints + * @description Inference endpoints exposed by this route + */ + supported_endpoints?: string[] | null; + /** Supported Modalities */ + supported_modalities?: string[] | null; + /** Supported Output Modalities */ + supported_output_modalities?: string[] | null; + /** Supports Function Calling */ + supports_function_calling?: boolean | null; + /** Supports Parallel Function Calling */ + supports_parallel_function_calling?: boolean | null; + /** Supports Reasoning */ + supports_reasoning?: boolean | null; } & { [key: string]: unknown; }; From 28f9f8bbcf60946b1e466df0e7c07dc021605ac4 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 5 Oct 2026 14:52:25 +0200 Subject: [PATCH 02/34] feat(router): share reasoning-aware output budget defaults --- examples/model_metadata/README.md | 18 ++++ litellm/llms/openai_like/model_info.py | 1 + litellm/proxy/model_discovery.py | 7 ++ litellm/router.py | 48 ++++++++-- .../router_utils/model_request_defaults.py | 79 +++++++++++++++ litellm/types/proxy/model_listing.py | 8 +- litellm/types/proxy/model_metadata.py | 40 +++++++- ..._utils_model_creation_and_error_logging.py | 4 + .../test_model_request_defaults.py | 96 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 11 +++ 10 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 litellm/router_utils/model_request_defaults.py create mode 100644 tests/unit/router_utils/test_model_request_defaults.py diff --git a/examples/model_metadata/README.md b/examples/model_metadata/README.md index df12d30f30d..7103f2d260e 100644 --- a/examples/model_metadata/README.md +++ b/examples/model_metadata/README.md @@ -40,3 +40,21 @@ Native Codex UI reasoning aliases are normalized to API effort values. `ultra` i For a model group with an explicit total context contract, discovery advertises the smallest known input, output, and total limit across its deployments. A limit is omitted if any deployment does not declare it. Capabilities require agreement across deployments, supported lists are intersected, and a default must agree across deployments. Existing input/output aggregation is preserved for legacy groups without a total context field Only configured models are exposed. Fetching a supplier's catalog does not add its other models to the downstream listing. Set `general_settings.disable_model_info_refresh: true` to disable upstream catalog polling + +Configured output budgets are request defaults, independent of supplier limits. Set `model_info.request_defaults` to advertise and apply the same policy centrally + +```yaml +model_info: + context_window: 262144 + request_defaults: + output_token_budget: 8192 + output_token_budget_by_reasoning_effort: + low: 65536 + high: 65536 + xhigh: 65536 + max: 131072 +``` + +The route chooses a configured budget from the effective request or deployment reasoning effort when the caller omits all explicit output budget parameters. An unmatched effort uses `output_token_budget`. Chat Completions and Anthropic Messages receive `max_tokens`, and Responses receives `max_output_tokens`. Caller-provided `max_tokens`, `max_completion_tokens`, or `max_output_tokens` takes precedence. The policy object is not sent to the supplier + +Discovery advertises `request_defaults` only when every deployment has the same policy. These chosen budgets do not populate `max_output_tokens`, establish a supplier maximum, or change the total context window. Clients can reserve the selected budget during compaction while the supplier's independent output limit remains unknown diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index ee3ae2c45c1..131bde0af27 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -107,6 +107,7 @@ class _ModelCard(GatewayModelMetadata): default_reasoning_effort=self.default_reasoning_effort or (self.reasoning.default_effort if provider == "openrouter" and self.reasoning is not None else None), supported_endpoints=self.supported_endpoints, + request_defaults=self.request_defaults, supported_modalities=self.supported_modalities if self.supported_modalities is not None else ( diff --git a/litellm/proxy/model_discovery.py b/litellm/proxy/model_discovery.py index 3430121353c..5fa0a71d6a3 100644 --- a/litellm/proxy/model_discovery.py +++ b/litellm/proxy/model_discovery.py @@ -43,6 +43,12 @@ def discover_model_metadata(sources: tuple[GatewayModelMetadata, ...]) -> ModelD defaults: Final = tuple(source.default_reasoning_effort for source in sources) default: Final = defaults[0] if defaults and all(value == defaults[0] for value in defaults) else None valid_default: Final = default if efforts is None or default in efforts else None + request_defaults: Final = tuple(source.request_defaults for source in sources) + common_request_defaults: Final = ( + request_defaults[0] + if request_defaults and all(value == request_defaults[0] for value in request_defaults) + else None + ) metadata: Final = GatewayModelMetadata( context_window=guaranteed_token_limit(sources, "context_window"), supports_function_calling=_common_boolean(tuple(source.supports_function_calling for source in sources)), @@ -55,5 +61,6 @@ def discover_model_metadata(sources: tuple[GatewayModelMetadata, ...]) -> ModelD supported_endpoints=_common_list(tuple(source.supported_endpoints for source in sources)), supported_modalities=_common_list(tuple(source.supported_modalities for source in sources)), supported_output_modalities=_common_list(tuple(source.supported_output_modalities for source in sources)), + request_defaults=common_request_defaults, ) return _ADAPTER.validate_python(metadata.model_dump(mode="json", exclude_none=True)) diff --git a/litellm/router.py b/litellm/router.py index f783bcb9cfe..7c035d1f2a5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -227,6 +227,7 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, warn_on_unknown_model_group_affinity_flags, @@ -2596,10 +2597,19 @@ class Router: ) # Check for silent model experiment # Make a local copy of litellm_params to avoid mutating the Router's state - litellm_params: Final = self._deployment_params_with_request_reasoning_override( + selected_params: Final = self._deployment_params_with_request_reasoning_override( deployment["litellm_params"], kwargs ) - silent_model: Final = litellm_params.pop("silent_model", None) + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), selected_params, kwargs, "completion" + ) + litellm_params: Final = deployment_params_with_request_defaults( + {key: value for key, value in selected_params.items() if key != "silent_model"}, + request_defaults, + deployment.get("model_info", {}), + kwargs, + ) + silent_model: Final = selected_params.get("silent_model") for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model @@ -2640,6 +2650,7 @@ class Router: "caching": self.cache_responses, "client": model_client, **kwargs, + **request_defaults, } response: Final = litellm.completion(**input_kwargs) verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -3745,10 +3756,19 @@ class Router: # Check for silent model experiment # Make a local copy of litellm_params to avoid mutating the Router's state - litellm_params: Final = self._deployment_params_with_request_reasoning_override( + selected_params: Final = self._deployment_params_with_request_reasoning_override( deployment["litellm_params"], kwargs ) - silent_model: Final = litellm_params.pop("silent_model", None) + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), selected_params, kwargs, "completion" + ) + litellm_params: Final = deployment_params_with_request_defaults( + {key: value for key, value in selected_params.items() if key != "silent_model"}, + request_defaults, + deployment.get("model_info", {}), + kwargs, + ) + silent_model: Final = selected_params.get("silent_model") for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model @@ -3779,6 +3799,7 @@ class Router: "caching": self.cache_responses, "client": model_client, **kwargs, + **request_defaults, } input_kwargs.pop("silent_model", None) input_kwargs.pop("include_fallback_errors", None) @@ -5380,9 +5401,17 @@ class Router: return await original_generic_function(model=model, **kwargs) raise e + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), + deployment["litellm_params"], + kwargs, + getattr(original_generic_function, "__name__", ""), + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name=function_name) - data: Final = deployment["litellm_params"].copy() + data: Final = deployment_params_with_request_defaults( + deployment["litellm_params"], request_defaults, deployment.get("model_info", {}), kwargs + ) model_name: Final = data["model"] self.total_calls[model_name] += 1 @@ -5398,6 +5427,7 @@ class Router: **kwargs, "model": model_name, **_with_router_resolved_session_model(kwargs.get("session"), model_name), + **request_defaults, } # Only set custom_llm_provider if it's not None if custom_llm_provider is not None: @@ -5850,9 +5880,14 @@ class Router: specific_deployment=kwargs.pop("specific_deployment", None), request_kwargs=kwargs, ) + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), deployment["litellm_params"], kwargs, handler_name + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="generic_api_call") - data: Final = deployment["litellm_params"].copy() + data: Final = deployment_params_with_request_defaults( + deployment["litellm_params"], request_defaults, deployment.get("model_info", {}), kwargs + ) model_name: Final = data["model"] self.total_calls[model_name] += 1 @@ -5874,6 +5909,7 @@ class Router: "custom_llm_provider": custom_llm_provider, "caching": self.cache_responses, **kwargs, + **request_defaults, } ) diff --git a/litellm/router_utils/model_request_defaults.py b/litellm/router_utils/model_request_defaults.py new file mode 100644 index 00000000000..b9b161270a0 --- /dev/null +++ b/litellm/router_utils/model_request_defaults.py @@ -0,0 +1,79 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.types.proxy.model_metadata import GatewayModelMetadata + +_METADATA_ADAPTER: Final = TypeAdapter(GatewayModelMetadata) +_EMPTY_DEFAULTS: Final[Mapping[str, int]] = MappingProxyType({}) +_OUTPUT_BUDGET_PARAMS: Final = frozenset(("max_tokens", "max_completion_tokens", "max_output_tokens")) + + +class _Reasoning(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + effort: str | int | None = None + + +class _RequestEffort(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + reasoning_effort: str | int | None = None + reasoning: _Reasoning | None = None + + def effort(self) -> str | int | None: + return ( + self.reasoning_effort + if self.reasoning_effort is not None + else self.reasoning.effort + if self.reasoning is not None + else None + ) + + +def model_request_defaults( + model_info: object, + deployment_params: Mapping[str, object], + request_params: Mapping[str, object], + call_type: str, +) -> Mapping[str, int]: + if call_type not in ("completion", "acompletion", "responses", "aresponses", "anthropic_messages"): + return _EMPTY_DEFAULTS + if any(request_params.get(key) is not None for key in _OUTPUT_BUDGET_PARAMS): + return _EMPTY_DEFAULTS + metadata: Final = _METADATA_ADAPTER.validate_python(model_info) + defaults: Final = metadata.request_defaults + if defaults is None: + return _EMPTY_DEFAULTS + requested_effort: Final = _RequestEffort.model_validate(request_params).effort() + deployment_effort: Final = _RequestEffort.model_validate(deployment_params).effort() + effort: Final = ( + requested_effort if requested_effort is not None else deployment_effort or metadata.default_reasoning_effort + ) + budget: Final = defaults.output_budget(str(effort) if effort is not None else None) + if budget is None: + return _EMPTY_DEFAULTS + key: Final = "max_output_tokens" if call_type in ("responses", "aresponses") else "max_tokens" + return MappingProxyType({key: budget}) + + +def deployment_params_with_request_defaults( + deployment_params: Mapping[str, object], + defaults: Mapping[str, int], + model_info: object = None, + request_params: Mapping[str, object] | None = None, +) -> Mapping[str, object]: + caller_budget: Final = request_params is not None and any( + request_params.get(key) is not None for key in _OUTPUT_BUDGET_PARAMS + ) + configured_policy: Final = ( + model_info is not None and _METADATA_ADAPTER.validate_python(model_info).request_defaults is not None + ) + replace_budget: Final = bool(defaults) or caller_budget and configured_policy + return MappingProxyType( + { + key: value + for key, value in deployment_params.items() + if not replace_budget or key not in _OUTPUT_BUDGET_PARAMS + } + ) diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index 8242ca991c5..b28c31c4449 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -1,4 +1,4 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence """Response types for the model listing/retrieve endpoints (/v1/models, /models).""" @@ -7,6 +7,11 @@ from typing import Literal from typing_extensions import NotRequired, ReadOnly, TypedDict +class ModelRequestDefaultsInfo(TypedDict, total=False): + output_token_budget: ReadOnly[int] + output_token_budget_by_reasoning_effort: ReadOnly[Mapping[str, int]] + + class ModelDiscoveryInfo(TypedDict, total=False): context_window: ReadOnly[int] supports_function_calling: ReadOnly[bool] @@ -17,6 +22,7 @@ class ModelDiscoveryInfo(TypedDict, total=False): supported_endpoints: ReadOnly[Sequence[str]] supported_modalities: ReadOnly[Sequence[str]] supported_output_modalities: ReadOnly[Sequence[str]] + request_defaults: ReadOnly[ModelRequestDefaultsInfo] class ModelInfoMetadata(TypedDict): diff --git a/litellm/types/proxy/model_metadata.py b/litellm/types/proxy/model_metadata.py index 0404bc770c1..8fe1639ca74 100644 --- a/litellm/types/proxy/model_metadata.py +++ b/litellm/types/proxy/model_metadata.py @@ -1,12 +1,45 @@ from collections.abc import Mapping, Sequence -from typing import Final +from types import MappingProxyType +from typing import Annotated, Final -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, ValidationInfo, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + TypeAdapter, + ValidationError, + ValidationInfo, + field_serializer, + field_validator, +) _STRING_VALUES_ADAPTER: Final = TypeAdapter(tuple[str, ...]) _VALIDATED_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +class ModelRequestDefaults(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + output_token_budget: Annotated[int, Field(gt=0, strict=True)] | None = None + output_token_budget_by_reasoning_effort: Mapping[str, Annotated[int, Field(gt=0, strict=True)]] | None = None + + @field_validator("output_token_budget_by_reasoning_effort") + @classmethod + def immutable_budgets(cls, value: Mapping[str, int] | None) -> Mapping[str, int] | None: + return MappingProxyType(dict(value)) if value is not None else None + + @field_serializer("output_token_budget_by_reasoning_effort") + def serialize_budgets(self, value: Mapping[str, int] | None) -> Mapping[str, int] | None: + return dict(value) if value is not None else None + + def output_budget(self, effort: str | None) -> int | None: + return ( + (self.output_token_budget_by_reasoning_effort or {}).get(effort, self.output_token_budget) + if effort is not None + else self.output_token_budget + ) + + class GatewayModelMetadata(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) @@ -27,6 +60,9 @@ class GatewayModelMetadata(BaseModel): ) supported_modalities: Sequence[str] | None = None supported_output_modalities: Sequence[str] | None = None + request_defaults: ModelRequestDefaults | None = Field( + default=None, description="Configured request defaults, independent of supplier capability limits" + ) @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") @classmethod diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 8b868db91f9..19a6e22fd37 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2448,6 +2448,10 @@ def test_model_listing_advertises_complete_configured_metadata_and_no_secrets(): "supported_endpoints": ["/v1/responses"], "supported_modalities": ["text", "image"], "supported_output_modalities": ["text"], + "request_defaults": { + "output_token_budget": 100, + "output_token_budget_by_reasoning_effort": {"xhigh": 150}, + }, } router: Final = litellm.Router( model_list=[ diff --git a/tests/unit/router_utils/test_model_request_defaults.py b/tests/unit/router_utils/test_model_request_defaults.py new file mode 100644 index 00000000000..71762c8dd8c --- /dev/null +++ b/tests/unit/router_utils/test_model_request_defaults.py @@ -0,0 +1,96 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.proxy.model_discovery import discover_model_metadata +from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults +from litellm.types.proxy.model_metadata import GatewayModelMetadata, ModelRequestDefaults + +_POLICY: Final = { + "output_token_budget": 8192, + "output_token_budget_by_reasoning_effort": {"low": 65536, "high": 65536, "xhigh": 65536, "max": 131072}, +} + + +@pytest.mark.parametrize( + "effort,expected", [(None, 8192), ("off", 8192), ("low", 65536), ("high", 65536), ("xhigh", 65536), ("max", 131072)] +) +def test_request_budget_follows_effective_reasoning_without_claiming_supplier_limit(effort, expected): + metadata: Final = {"context_window": 262144, "request_defaults": _POLICY} + request: Final = {"reasoning_effort": effort} + assert model_request_defaults(metadata, {}, request, "completion") == {"max_tokens": expected} + assert "max_output_tokens" not in discover_model_metadata((GatewayModelMetadata.model_validate(metadata),)) + assert request == {"reasoning_effort": effort} + + +@pytest.mark.parametrize("key", ["max_tokens", "max_completion_tokens", "max_output_tokens"]) +def test_explicit_caller_budget_wins(key): + assert ( + model_request_defaults({"request_defaults": _POLICY}, {}, {key: 123, "reasoning_effort": "max"}, "completion") + == {} + ) + + +@pytest.mark.parametrize( + "surface,key", + [ + ("completion", "max_tokens"), + ("acompletion", "max_tokens"), + ("responses", "max_output_tokens"), + ("aresponses", "max_output_tokens"), + ("anthropic_messages", "max_tokens"), + ], +) +def test_budget_uses_endpoint_parameter_and_request_reasoning_overrides_deployment(surface, key): + assert model_request_defaults( + {"request_defaults": _POLICY}, + {"reasoning_effort": "max", "max_tokens": 456}, + {"reasoning": {"effort": "low"}}, + surface, + ) == {key: 65536} + + +def test_deployment_and_metadata_default_efforts_are_resolved(): + metadata: Final = {"request_defaults": _POLICY, "default_reasoning_effort": "high"} + assert model_request_defaults(metadata, {}, {}, "completion") == {"max_tokens": 65536} + assert model_request_defaults(metadata, {"reasoning": {"effort": "max"}}, {}, "responses") == { + "max_output_tokens": 131072 + } + assert model_request_defaults(metadata, {}, {}, "aembedding") == {} + assert model_request_defaults({}, {}, {}, "completion") == {} + assert model_request_defaults(metadata, {}, {"reasoning_effort": 50}, "completion") == {"max_tokens": 8192} + + +def test_selected_policy_replaces_competing_static_budget_aliases(): + deployment: Final = {"model": "openai/fixture", "max_tokens": 1, "max_completion_tokens": 2, "max_output_tokens": 3} + defaults: Final = model_request_defaults( + {"request_defaults": _POLICY}, deployment, {"reasoning_effort": "max"}, "responses" + ) + params: Final = deployment_params_with_request_defaults(deployment, defaults) + assert {**params, **defaults} == {"model": "openai/fixture", "max_output_tokens": 131072} + assert deployment_params_with_request_defaults(deployment, {}) == deployment + assert deployment_params_with_request_defaults( + deployment, {}, {"request_defaults": _POLICY}, {"max_tokens": 123} + ) == {"model": "openai/fixture"} + assert deployment["max_completion_tokens"] == 2 + + +@pytest.mark.parametrize("value", [0, -1, True, "8192", 1.5]) +@pytest.mark.parametrize("field", ["output_token_budget", "output_token_budget_by_reasoning_effort"]) +def test_invalid_request_budgets_are_rejected(value, field): + invalid: Final = {field: {"high": value} if field.endswith("effort") else value} + with pytest.raises(ValidationError): + ModelRequestDefaults.model_validate(invalid) + + +def test_request_budget_maps_are_immutable_and_only_common_policy_is_advertised(): + metadata: Final = GatewayModelMetadata.model_validate({"request_defaults": _POLICY}) + assert discover_model_metadata((metadata, metadata)) == {"request_defaults": _POLICY} + assert discover_model_metadata((metadata, GatewayModelMetadata())) == {} + other: Final = GatewayModelMetadata(request_defaults=ModelRequestDefaults(output_token_budget=16384)) + assert discover_model_metadata((metadata, other)) == {} + defaults: Final = metadata.request_defaults + assert defaults is not None + with pytest.raises(TypeError): + defaults.output_token_budget_by_reasoning_effort["max"] = 1 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c9903caf911..5c1107073a3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38425,6 +38425,15 @@ export interface components { */ purpose: "extract" | "cluster" | "investigate"; }; + /** ModelRequestDefaults */ + ModelRequestDefaults: { + /** Output Token Budget */ + output_token_budget?: number | null; + /** Output Token Budget By Reasoning Effort */ + output_token_budget_by_reasoning_effort?: { + [key: string]: number; + } | null; + }; /** ModelResponse */ ModelResponse: { /** Choices */ @@ -50339,6 +50348,8 @@ export interface components { * @description Exact effort values this route accepts */ reasoning_effort_levels?: string[] | null; + /** @description Configured request defaults, independent of supplier capability limits */ + request_defaults?: components["schemas"]["ModelRequestDefaults"] | null; /** * Supported Endpoints * @description Inference endpoints exposed by this route From 8c21000cdf836dbbdda01e36d9270b867d1874ec Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 5 Oct 2026 16:36:40 +0200 Subject: [PATCH 03/34] feat: reload external model offerings from supplier inventories --- examples/model_offerings/README.md | 62 ++++ litellm/llms/chatgpt/authenticator.py | 16 +- litellm/llms/chatgpt/chat/transformation.py | 2 +- litellm/llms/chatgpt/model_info.py | 55 ++- litellm/llms/openai_like/model_info.py | 78 ++++- litellm/proxy/_types.py | 3 + litellm/proxy/model_offerings.py | 327 ++++++++++++++++++ litellm/proxy/offering_router.py | 159 +++++++++ litellm/proxy/proxy_server.py | 54 ++- litellm/proxy/utils.py | 9 +- litellm/router.py | 20 ++ litellm/types/proxy/model_inventory.py | 15 + litellm/types/proxy/model_offerings.py | 106 ++++++ .../llms/chatgpt/chat/test_transformation.py | 21 ++ .../chatgpt/test_chatgpt_authenticator.py | 10 + .../unit/llms/openai_like/test_model_info.py | 32 ++ tests/unit/proxy/test_model_offerings.py | 164 +++++++++ tests/unit/proxy/test_offering_router.py | 45 +++ .../unit/types/proxy/test_model_offerings.py | 42 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 20 files changed, 1194 insertions(+), 31 deletions(-) create mode 100644 examples/model_offerings/README.md create mode 100644 litellm/proxy/model_offerings.py create mode 100644 litellm/proxy/offering_router.py create mode 100644 litellm/types/proxy/model_inventory.py create mode 100644 litellm/types/proxy/model_offerings.py create mode 100644 tests/unit/llms/chatgpt/chat/test_transformation.py create mode 100644 tests/unit/proxy/test_model_offerings.py create mode 100644 tests/unit/proxy/test_offering_router.py create mode 100644 tests/unit/types/proxy/test_model_offerings.py diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md new file mode 100644 index 00000000000..db8cbff25a9 --- /dev/null +++ b/examples/model_offerings/README.md @@ -0,0 +1,62 @@ +# External model offerings + +Set `general_settings.model_offerings_path` in the normal proxy configuration and keep its `model_list` empty. Disable `store_model_in_db`; the database may still serve authentication and usage tracking. Existing proxy configurations behave as before when the path is absent + +```yaml +model_list: [] +general_settings: + model_offerings_path: /app/offerings/offerings.yaml + store_model_in_db: false +``` + +Mount a host directory into Docker, for example `./offerings:/app/offerings:ro`. Keep `offerings.yaml` inside it. Replace the file atomically within that directory so the container sees the new inode. A bind mount of an individual file does not reliably see atomic replacement + +```yaml +version: 1 +config_poll_seconds: 5 +inventory_poll_seconds: 300 +providers: + openrouter: + provider: openrouter + api_key: os.environ/OPENROUTER_API_KEY + vercel: + provider: vercel_ai_gateway + api_key: os.environ/VERCEL_API_KEY + local: + provider: openai + api_base: http://inference-server:8000/v1 + api_key: os.environ/INFERENCE_API_KEY + subscribed: + provider: chatgpt +offerings: + - model_name: selected + source: auto + provider: openrouter + upstream_model: organization/model + enabled: true + - model_name: handwritten + source: manual + provider: openrouter + upstream_model: organization/another-model + model_info: + mode: chat + context_window: 262144 + supports_function_calling: true + request_defaults: + output_token_budget: 8192 + output_token_budget_by_reasoning_effort: + low: 65536 + high: 65536 + xhigh: 65536 + max: 131072 +``` + +These example token values illustrate operator configuration, not a claim about the example supplier models. Request defaults express chosen budgets and do not populate independent output capability limits + +`source` belongs to each offering. AUTO entries require membership in the latest successful full inventory for their connection. Missing entries disappear from model listings and return a useful unavailable error before forwarding. A returning entry becomes available again. MANUAL entries use only handwritten definitions and are independent of supplier inventory membership. Both sources accept shared `model_info` overrides; explicit overrides take precedence over downloaded metadata. Wildcards and duplicate public aliases are rejected + +Startup validates the complete file and awaits initial inventory requests. AUTO entries remain unavailable when no successful inventory exists; MANUAL entries remain usable. A successful empty inventory means absence. HTTP errors, authentication failures, timeouts and malformed responses retain the last successful inventory for unchanged credentials. Changing connection credentials starts with unknown availability and forces a fresh inventory. ChatGPT uses its existing native OAuth account and cannot initiate device login from discovery or request handling + +The file is checked every five seconds by default, with a configurable range of 1 to 300 seconds. Inventories refresh every 300 seconds by default, with a range of 1 to 86400 seconds. Invalid file edits retain the whole last-valid serving configuration, including credentials, and log only the error kind. Unselected inventory entries never become public routes. Caller connection overrides and runtime model-management mutations are rejected in external mode; edit the external file instead + +Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The inventory HTTP client and polling task are closed at proxy shutdown after requests drain diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index 20c625fabcb..0d4ec65c881 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -2,7 +2,9 @@ import base64 import json import os import time -from collections.abc import Mapping +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar from typing import Final, TypeAlias import httpx @@ -34,6 +36,16 @@ OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth" JsonObject: TypeAlias = Mapping[str, JsonValue] _JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject) +_DEVICE_LOGIN_ENABLED: Final[ContextVar[bool]] = ContextVar("chatgpt_device_login_enabled", default=True) + + +@contextmanager +def prevent_device_login() -> Generator[None]: + token: Final = _DEVICE_LOGIN_ENABLED.set(False) + try: + yield + finally: + _DEVICE_LOGIN_ENABLED.reset(token) def _optional_str(value: JsonValue | None) -> str | None: @@ -66,7 +78,7 @@ class Authenticator: except RefreshAccessTokenError as exc: verbose_logger.warning("ChatGPT refresh token failed, re-login required: %s", exc) - if not allow_device_login: + if not allow_device_login or not _DEVICE_LOGIN_ENABLED.get(): raise GetAccessTokenError( status_code=401, message="ChatGPT authorization is unavailable; model discovery cannot start device login", diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index 1b110704c8b..1583179cd95 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -33,7 +33,7 @@ class ChatGPTConfig(OpenAIConfig): api_key: str | None, custom_llm_provider: str, ) -> tuple[str | None, str | None, str]: - dynamic_api_base: Final = self.api_base_without_login() + dynamic_api_base: Final = api_base or self.api_base_without_login() try: dynamic_api_key: Final = self.authenticator.get_access_token() except GetAccessTokenError as e: diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index 36bba20c7a5..46e306a7e25 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -6,12 +6,12 @@ from types import MappingProxyType from typing import Final import httpx -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator -from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory from litellm.types.proxy.model_metadata import GatewayModelMetadata from .authenticator import Authenticator @@ -105,7 +105,8 @@ class _ChatGPTModel(GatewayModelMetadata): class _ChatGPTCatalog(BaseModel): model_config = ConfigDict(frozen=True) - models: tuple[_ChatGPTModel, ...] = () + models: tuple[_ChatGPTModel, ...] + error: object | None = None async def get_chatgpt_model_info( @@ -116,11 +117,36 @@ async def get_chatgpt_model_info( api_base: str | None = None, authenticator: Authenticator | None = None, ) -> Mapping[str, object]: + inventory: Final = await get_chatgpt_model_inventory( + client=client, cache=cache, api_base=api_base, authenticator=authenticator + ) + return ( + inventory.models.get(model, _EMPTY_METADATA) + if isinstance(inventory, SupplierModelInventory) + else _EMPTY_METADATA + ) + + +async def get_chatgpt_model_inventory( + *, + client: AsyncHTTPHandler, + cache: InMemoryCache, + api_base: str | None = None, + authenticator: Authenticator | None = None, + force_refresh: bool = False, +) -> SupplierModelInventory | SupplierInventoryUnavailable: auth: Final = authenticator if authenticator is not None else Authenticator() try: - token: Final = await asyncio.to_thread(auth.get_access_token, allow_device_login=False) account: Final = await asyncio.to_thread(auth.get_account_id) url: Final = f"{(api_base or auth.get_api_base()).rstrip('/')}/models?client_version={_CATALOG_CLIENT_VERSION}" + credential_scope: Final = hashlib.sha256(json.dumps((url, account)).encode()).hexdigest() if account else None + except Exception: # noqa: BLE001 # malformed native auth is unavailable, never an authoritative empty inventory + return SupplierInventoryUnavailable("authentication") + try: + token: Final = await asyncio.to_thread(auth.get_access_token, allow_device_login=False) + except Exception: # noqa: BLE001 # unavailable existing OAuth must never initiate a device login + return SupplierInventoryUnavailable("authentication", credential_scope) + try: headers: Final = { **get_chatgpt_default_headers(token, account), "accept": "application/json", @@ -129,8 +155,8 @@ async def get_chatgpt_model_info( "chatgpt_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() ) cached: Final[object] = cache.get_cache(cache_key) - if isinstance(cached, _ChatGPTCatalog): - return next((card.metadata() for card in cached.models if card.slug == model), _EMPTY_METADATA) + if not force_refresh and isinstance(cached, SupplierModelInventory): + return cached response: Final = await client.get( url=url, headers=headers, @@ -140,8 +166,17 @@ async def get_chatgpt_model_info( ) response.raise_for_status() catalog: Final = _ChatGPTCatalog.model_validate_json(response.content) - cache.set_cache(cache_key, catalog, ttl=MODEL_INFO_REFRESH_SECONDS) - return next((card.metadata() for card in catalog.models if card.slug == model), _EMPTY_METADATA) + ids: Final = tuple(card.slug for card in catalog.models) + if catalog.error is not None or any(not model_id for model_id in ids) or len(frozenset(ids)) != len(ids): + return SupplierInventoryUnavailable("malformed", credential_scope) + inventory: Final = SupplierModelInventory( + MappingProxyType({card.slug: card.metadata() for card in catalog.models}), credential_scope + ) + cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) + return inventory + except httpx.HTTPStatusError: + return SupplierInventoryUnavailable("http", credential_scope) + except ValidationError: + return SupplierInventoryUnavailable("malformed", credential_scope) except Exception: # noqa: BLE001 # optional metadata discovery must not interrupt proxy startup - verbose_logger.debug("Could not discover the active ChatGPT account model metadata") - return _EMPTY_METADATA + return SupplierInventoryUnavailable("transport", credential_scope) diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 131bde0af27..70966605204 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -5,11 +5,11 @@ from types import MappingProxyType from typing import Annotated, Final, TypeAlias import httpx -from pydantic import BaseModel, BeforeValidator, ConfigDict +from pydantic import BaseModel, BeforeValidator, ConfigDict, ValidationError -from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.utils import _add_path_to_api_base # pyright: ignore[reportPrivateUsage] # shared provider URL helper @@ -55,6 +55,8 @@ class _ReasoningOption(BaseModel): class _ModelCard(GatewayModelMetadata): id: str + type: str | None = None + mode: str | None = None max_model_len: _TokenLimit = None context_length: _TokenLimit = None max_tokens: _TokenLimit = None @@ -81,7 +83,7 @@ class _ModelCard(GatewayModelMetadata): max_input_tokens=self.max_input_tokens, max_output_tokens=( self.max_output_tokens - or (self.max_tokens if provider == "vercel_ai_gateway" else None) + or (self.max_tokens if provider == "vercel_ai_gateway" and self.type in (None, "language") else None) or ( self.top_provider.max_completion_tokens if provider == "openrouter" and self.top_provider is not None @@ -127,13 +129,27 @@ class _ModelCard(GatewayModelMetadata): else None ), ) - return MappingProxyType(metadata.model_dump(mode="json", exclude_none=True)) + mode: Final = ( + self.mode + if self.mode in ("chat", "completion", "embedding") + else ( + "embedding" + if provider == "vercel_ai_gateway" and self.type == "embedding" + else "chat" + if provider == "vercel_ai_gateway" and self.type == "language" + else None + ) + ) + return MappingProxyType( + {**metadata.model_dump(mode="json", exclude_none=True), **({"mode": mode} if mode is not None else {})} + ) class _ModelList(BaseModel): model_config = ConfigDict(frozen=True) - data: tuple[_ModelCard, ...] = () + data: tuple[_ModelCard, ...] + error: object | None = None async def get_openai_compatible_model_info( @@ -145,13 +161,31 @@ async def get_openai_compatible_model_info( client: AsyncHTTPHandler, cache: InMemoryCache, ) -> Mapping[str, object]: + inventory: Final = await get_openai_compatible_model_inventory( + api_base=api_base, provider=provider, headers=headers, client=client, cache=cache + ) + return ( + inventory.models.get(model, _EMPTY_LIMITS) if isinstance(inventory, SupplierModelInventory) else _EMPTY_LIMITS + ) + + +async def get_openai_compatible_model_inventory( + *, + api_base: str, + provider: str = "openai", + headers: Mapping[str, str], + client: AsyncHTTPHandler, + cache: InMemoryCache, + force_refresh: bool = False, +) -> SupplierModelInventory | SupplierInventoryUnavailable: url: Final = _add_path_to_api_base(api_base, "/v1/models") cache_key: Final = ( - "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() + "upstream_model_info:" + + hashlib.sha256(json.dumps((provider, url, sorted(headers.items()))).encode()).hexdigest() ) cached: Final[object] = cache.get_cache(cache_key) - if isinstance(cached, _ModelList): - return next((card.token_limits(provider) for card in cached.data if card.id == model), _EMPTY_LIMITS) + if not force_refresh and isinstance(cached, (SupplierModelInventory, SupplierInventoryUnavailable)): + return cached try: response: Final = await client.get( @@ -163,10 +197,26 @@ async def get_openai_compatible_model_info( ) response.raise_for_status() models: Final = _ModelList.model_validate_json(response.content) - except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh - verbose_logger.debug("Could not discover upstream model token limits") - cache.set_cache(cache_key, _ModelList(), ttl=60) - return _EMPTY_LIMITS + ids: Final = tuple(card.id for card in models.data) + if models.error is not None or any(not model_id for model_id in ids) or len(frozenset(ids)) != len(ids): + unavailable: Final = SupplierInventoryUnavailable("malformed") + cache.set_cache(cache_key, unavailable, ttl=60) + return unavailable + except httpx.HTTPStatusError: + http_failure: Final = SupplierInventoryUnavailable("http") + cache.set_cache(cache_key, http_failure, ttl=60) + return http_failure + except ValidationError: + malformed: Final = SupplierInventoryUnavailable("malformed") + cache.set_cache(cache_key, malformed, ttl=60) + return malformed + except Exception: # noqa: BLE001 # optional upstream discovery must not interrupt proxy refresh + transport: Final = SupplierInventoryUnavailable("transport") + cache.set_cache(cache_key, transport, ttl=60) + return transport - cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS) - return next((card.token_limits(provider) for card in models.data if card.id == model), _EMPTY_LIMITS) + inventory: Final = SupplierModelInventory( + MappingProxyType({card.id: card.token_limits(provider) for card in models.data}) + ) + cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) + return inventory diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index cf7536182a5..e66b2563dc7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2727,6 +2727,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): Documents all the fields supported by `general_settings` in config.yaml """ + model_offerings_path: str | None = Field( + None, description="Path to the authoritative external offering file, reloaded without restarting the proxy" + ) completion_model: str | None = Field(None, description="proxy level default model for all chat completion calls") max_in_flight_requests_per_worker: int | None = Field( None, gt=0, description="maximum concurrent requests handled by each worker" diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py new file mode 100644 index 00000000000..ddf4c5361db --- /dev/null +++ b/litellm/proxy/model_offerings.py @@ -0,0 +1,327 @@ +import asyncio +import hashlib +import json +import os +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import yaml + +from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.chatgpt.authenticator import Authenticator, prevent_device_login +from litellm.llms.chatgpt.model_info import get_chatgpt_model_inventory +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY, get_openai_compatible_model_inventory +from litellm.proxy.offering_router import OfferingRouterView, OfferingServingSnapshot +from litellm.router import Router +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.types.proxy.model_offerings import ModelOffering, ModelOfferingsConfig, SupplierConnection + + +@dataclass(frozen=True, slots=True) +class _ResolvedConnection: + provider: str + api_base: str | None + api_key: str | None + headers: Mapping[str, str] + fingerprint: str + native_account_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class _ProviderState: + connection: _ResolvedConnection + inventory: SupplierModelInventory | None + credential_scope: str | None = None + + +@dataclass(frozen=True, slots=True) +class _LoadedConfig: + config: ModelOfferingsConfig + fingerprint: str + connections: Mapping[str, _ResolvedConnection] + + +@dataclass(frozen=True, slots=True) +class _InvalidConfig: + error_kind: str + + +def _resolve_secret(value: str) -> str: + if not value.startswith("os.environ/"): + return value + resolved: Final = os.environ.get(value.removeprefix("os.environ/")) + if resolved is None: + raise ValueError("Offering connection references an unset environment variable") + return resolved + + +def _native_account_id() -> str | None: + try: + return Authenticator().get_account_id() + except Exception: # noqa: BLE001 # an unreadable native auth file is not proof of an account change + return None + + +def _resolve_connection(connection: SupplierConnection) -> _ResolvedConnection: + api_base: Final = ( + str(connection.api_base).rstrip("/") + if connection.api_base is not None + else { + "openrouter": "https://openrouter.ai/api/v1", + "vercel_ai_gateway": "https://ai-gateway.vercel.sh/v1", + }.get(connection.provider) + ) + api_key: Final = _resolve_secret(connection.api_key.get_secret_value()) if connection.api_key is not None else None + configured_headers: Final = { + name.lower(): _resolve_secret(value.get_secret_value()) for name, value in connection.headers.items() + } + headers: Final = MappingProxyType( + {**({"authorization": f"Bearer {api_key}"} if api_key is not None else {}), **configured_headers} + ) + fingerprint: Final = hashlib.sha256( + json.dumps((connection.provider, api_base, api_key, sorted(headers.items()))).encode() + ).hexdigest() + return _ResolvedConnection( + connection.provider, + api_base, + api_key, + headers, + fingerprint, + _native_account_id() if connection.provider == "chatgpt" else None, + ) + + +def _load_config(path: Path) -> _LoadedConfig | _InvalidConfig: + try: + content: Final = path.read_bytes() + if len(content) > 1024 * 1024: + return _InvalidConfig("ConfigTooLarge") + config: Final = ModelOfferingsConfig.model_validate(yaml.safe_load(content)) + connections: Final = MappingProxyType( + {name: _resolve_connection(connection) for name, connection in config.providers.items()} + ) + fingerprint: Final = hashlib.sha256( + json.dumps( + ( + hashlib.sha256(content).hexdigest(), + tuple( + (connection.fingerprint, connection.native_account_id) for connection in connections.values() + ), + ) + ).encode() + ).hexdigest() + return _LoadedConfig(config, fingerprint, connections) + except Exception as error: # noqa: BLE001 # a bad operator edit must leave the complete last-valid snapshot intact + return _InvalidConfig(type(error).__name__) + + +def _provider_after_fetch( + previous: _ProviderState, fetched: SupplierModelInventory | SupplierInventoryUnavailable +) -> _ProviderState: + if isinstance(fetched, SupplierModelInventory): + return _ProviderState(previous.connection, fetched, fetched.credential_scope) + if fetched.credential_scope is not None and previous.credential_scope != fetched.credential_scope: + return _ProviderState(previous.connection, None, fetched.credential_scope) + return previous + + +def _unavailable_reason(offering: ModelOffering, state: _ProviderState) -> str | None: + if not offering.enabled: + return "disabled in the offering configuration" + if offering.source == "manual": + return None + if state.inventory is None: + return "no successful inventory is available for the configured supplier connection" + if offering.upstream_model not in state.inventory.models: + return "the model is absent from the latest successful supplier inventory" + return None + + +def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, object]: + connection: Final = state.connection + reason: Final = _unavailable_reason(offering, state) + supplier_info: Final[Mapping[str, object]] = ( + state.inventory.models.get(offering.upstream_model, {}) + if offering.source == "auto" and state.inventory is not None + else {} + ) + metadata: Final = {**supplier_info, **offering.model_info.model_dump(mode="json", exclude_none=True)} + deployment_id: Final = ( + "offering-" + + hashlib.sha256( + json.dumps((offering.model_name, connection.fingerprint, offering.upstream_model)).encode() + ).hexdigest() + ) + return MappingProxyType( + { + "model_name": offering.model_name, + "litellm_params": { + "model": f"{connection.provider}/{offering.upstream_model}", + **({"api_base": connection.api_base} if connection.api_base is not None else {}), + **({"api_key": connection.api_key} if connection.api_key is not None else {}), + **({"extra_headers": dict(connection.headers)} if connection.headers else {}), + }, + "model_info": {**metadata, "id": deployment_id, "blocked": reason is not None}, + } + ) + + +class ModelOfferingsManager: + def __init__( + self, + *, + path: Path, + template: Router, + client: AsyncHTTPHandler, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.path = path + self.template = template + self.client = client + self.clock = clock + self.cache = InMemoryCache(max_size_in_memory=64) + self.config: ModelOfferingsConfig | None = None + self.config_fingerprint: str | None = None + self.providers: Mapping[str, _ProviderState] = MappingProxyType({}) + self.deployments: tuple[Mapping[str, object], ...] = () + self.next_inventory_refresh = 0.0 + self.last_error: str | None = None + self.lock = asyncio.Lock() + self.inventory_semaphore = asyncio.Semaphore(MODEL_INFO_REFRESH_CONCURRENCY) + self.router = OfferingRouterView( + OfferingServingSnapshot(template, frozenset(), MappingProxyType({}), frozenset()) + ) + + async def _fetch_provider(self, state: _ProviderState) -> _ProviderState: + connection: Final = state.connection + try: + fetched: Final = ( + await get_chatgpt_model_inventory( + client=self.client, cache=self.cache, api_base=connection.api_base, force_refresh=True + ) + if connection.provider == "chatgpt" + else await get_openai_compatible_model_inventory( + api_base=connection.api_base or "", + provider=connection.provider, + headers=connection.headers, + client=self.client, + cache=self.cache, + force_refresh=True, + ) + ) + except Exception: # noqa: BLE001 # supplier failures cannot replace a successful inventory with absence + return state + if isinstance(fetched, SupplierInventoryUnavailable): + verbose_proxy_logger.warning( + "Supplier inventory refresh unavailable (%s); retaining the last successful inventory when connection identity is unchanged", + fetched.reason, + ) + return _provider_after_fetch(state, fetched) + + async def _bounded_fetch_provider(self, state: _ProviderState) -> _ProviderState: + async with self.inventory_semaphore: + return await self._fetch_provider(state) + + async def _refresh_providers(self, providers: Mapping[str, _ProviderState]) -> Mapping[str, _ProviderState]: + names: Final = tuple(providers) + results: Final = await asyncio.gather(*(self._bounded_fetch_provider(providers[name]) for name in names)) + return MappingProxyType(dict(zip(names, results))) + + def _apply(self, loaded: _LoadedConfig, providers: Mapping[str, _ProviderState]) -> bool: + deployments: Final = tuple( + _deployment(offering, providers[offering.provider]) + for offering in loaded.config.offerings + if _unavailable_reason(offering, providers[offering.provider]) is None + ) + try: + with prevent_device_login(): + candidate: Final = ( + self.template.snapshot_with_model_list(deployments, metadata_authoritative=True) + if deployments != self.deployments + else self.router.latest_snapshot().router + ) + if len(candidate.get_model_ids()) != len(deployments): + self._config_error("DeploymentValidationFailed") + return False + except Exception as error: # noqa: BLE001 # a rejected native deployment must not partially replace serving state + self._config_error(type(error).__name__) + return False + available: Final = frozenset( + offering.model_name + for offering in loaded.config.offerings + if _unavailable_reason(offering, providers[offering.provider]) is None + ) + unavailable: Final = MappingProxyType( + { + offering.model_name: reason + for offering in loaded.config.offerings + if (reason := _unavailable_reason(offering, providers[offering.provider])) is not None + } + ) + allowed_deployments: Final = frozenset( + f"{providers[offering.provider].connection.provider}/{offering.upstream_model}" + for offering in loaded.config.offerings + if offering.model_name in available + ) + snapshot: Final = OfferingServingSnapshot(candidate, available, unavailable, allowed_deployments) + self.config = loaded.config + self.config_fingerprint = loaded.fingerprint + self.providers = providers + self.deployments = deployments + self.last_error = None + self.router.publish_snapshot(snapshot) + return True + + def _config_error(self, kind: str) -> None: + if self.last_error != kind: + verbose_proxy_logger.error( + "External offering configuration rejected (%s); retaining the complete last-valid serving configuration", + kind, + ) + self.last_error = kind + + async def reload(self, *, initial: bool = False, force_inventory: bool = False) -> bool: + async with self.lock: + loaded: Final = await asyncio.to_thread(_load_config, self.path) + if isinstance(loaded, _InvalidConfig): + self._config_error(loaded.error_kind) + return False + now: Final = self.clock() + config_changed: Final = loaded.fingerprint != self.config_fingerprint + if not config_changed and not force_inventory and now < self.next_inventory_refresh: + return True + providers: Final = MappingProxyType( + { + name: previous + if (previous := self.providers.get(name)) is not None + and previous.connection.fingerprint == connection.fingerprint + and ( + connection.native_account_id is None + or previous.connection.native_account_id == connection.native_account_id + ) + else _ProviderState(connection, None) + for name, connection in loaded.connections.items() + } + ) + connection_changed: Final = any( + name not in self.providers or self.providers[name].connection != state.connection + for name, state in providers.items() + ) + refresh_due: Final = initial or force_inventory or connection_changed or now >= self.next_inventory_refresh + refreshed: Final = await self._refresh_providers(providers) if refresh_due else providers + if not self._apply(loaded, refreshed): + return False + if refresh_due: + self.next_inventory_refresh = now + loaded.config.inventory_poll_seconds + return True + + async def run(self) -> None: + while True: + await asyncio.sleep(self.config.config_poll_seconds if self.config is not None else 5) + await self.reload() diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py new file mode 100644 index 00000000000..ae46285177b --- /dev/null +++ b/litellm/proxy/offering_router.py @@ -0,0 +1,159 @@ +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Final, cast # noqa: TID251 # native Router dynamic attribute dispatch + +from starlette.types import ASGIApp, Receive, Scope, Send + +import litellm +from litellm.caching.caching import DualCache +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.proxy._types import UserAPIKeyAuth +from litellm.router import Router +from litellm.types.utils import CallTypes, CallTypesLiteral + + +@dataclass(frozen=True, slots=True) +class OfferingServingSnapshot: + router: Router + available_models: frozenset[str] + unavailable_models: Mapping[str, str] + allowed_deployments: frozenset[str] + + +_PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] = ContextVar( + "model_offering_snapshot", default=None +) +_MODEL_MUTATIONS: Final = frozenset(("set_model_list", "add_deployment", "upsert_deployment", "delete_deployment")) +_VIEW_MEMBERS: Final = frozenset(("publish_snapshot", "pin_snapshot", "serving_snapshot", "latest_snapshot")) + + +class OfferingRouterView(Router): + def __init__(self, snapshot: OfferingServingSnapshot) -> None: + self._offering_snapshot = snapshot + + def __getattribute__(self, name: str) -> object: + if name.startswith("_offering_") or name in _VIEW_MEMBERS: + return cast(object, object.__getattribute__(self, name)) # cast-ok: native Router dispatch + if name in _MODEL_MUTATIONS: + return cast( # cast-ok: native Router dispatch + object, object.__getattribute__(self, "_offering_reject_mutation") + ) + snapshot: Final = self.serving_snapshot() + if name == "model_names": + return snapshot.available_models | frozenset(snapshot.unavailable_models) + return cast(object, getattr(snapshot.router, name)) # cast-ok: dynamic native Router compatibility boundary + + def __setattr__(self, name: str, value: object) -> None: + if name.startswith("_offering_"): + object.__setattr__(self, name, value) + return + setattr(self.serving_snapshot().router, name, value) + + def _offering_reject_mutation(self, *args: object, **kwargs: object) -> None: + raise litellm.BadRequestError( + message="Model definitions are owned by the external offering configuration", model="", llm_provider="" + ) + + def serving_snapshot(self) -> OfferingServingSnapshot: + pinned: Final = _PINNED_SNAPSHOT.get() + return pinned[1] if pinned is not None and pinned[0] == id(self) else self._offering_snapshot + + def latest_snapshot(self) -> OfferingServingSnapshot: + return self._offering_snapshot + + def publish_snapshot(self, snapshot: OfferingServingSnapshot) -> None: + self._offering_snapshot = snapshot + + @contextmanager + def pin_snapshot(self) -> Generator[None]: + token: Final = _PINNED_SNAPSHOT.set((id(self), self._offering_snapshot)) + try: + with prevent_device_login(): + yield + finally: + _PINNED_SNAPSHOT.reset(token) + + +class OfferingSnapshotMiddleware: + def __init__(self, app: ASGIApp, router_getter: Callable[[], Router | None]) -> None: + self.app = app + self.router_getter = router_getter + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + router: Final = self.router_getter() + if scope["type"] != "http" or not isinstance(router, OfferingRouterView): + await self.app(scope, receive, send) + return + with router.pin_snapshot(): + await self.app(scope, receive, send) + + +class OfferingAccessGuard(CustomLogger): + def __init__(self, router: OfferingRouterView) -> None: + self.router = router + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + snapshot: Final = self.router.serving_snapshot() + return tuple(name for name in model_names if name in snapshot.available_models) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: Mapping[str, object], + call_type: CallTypesLiteral, + ) -> Exception | None: + if any( + data.get(name) is not None + for name in ( + "api_base", + "base_url", + "api_key", + "user_config", + "custom_llm_provider", + "extra_headers", + "headers", + "provider_specific_header", + ) + ): + return litellm.BadRequestError( + message="External offering mode does not accept caller-defined supplier connections", + model="", + llm_provider="", + ) + model: Final = data.get("model") + if not isinstance(model, str): + return None + snapshot: Final = self.router.serving_snapshot() + if model in snapshot.available_models: + return None + reason: Final = snapshot.unavailable_models.get(model) + if reason is not None: + return litellm.ServiceUnavailableError( + message=f"Model '{model}' is unavailable: {reason}", model=model, llm_provider="" + ) + return litellm.NotFoundError( + message=f"Model '{model}' is not an enabled gateway offering", model=model, llm_provider="" + ) + + async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: CallTypes | None) -> None: + model: Final = kwargs.get("model") + if not isinstance(model, str) or call_type not in ( + CallTypes.acompletion, + CallTypes.completion, + CallTypes.aresponses, + CallTypes.responses, + CallTypes.anthropic_messages, + CallTypes.aembedding, + CallTypes.embedding, + ): + return + if model not in self.router.serving_snapshot().allowed_deployments: + raise litellm.NotFoundError( + message="The requested deployment is not an enabled gateway offering", model=model, llm_provider="" + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9105c3078a9..a7ee7470c83 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1587,11 +1587,47 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() + import pathlib + + from litellm.proxy.model_offerings import ModelOfferingsManager + from litellm.proxy.offering_router import OfferingAccessGuard + + offerings_path: Final = TypeAdapter(str | None).validate_python( + cast(object, general_settings.get("model_offerings_path")), # cast-ok: validate untyped config + strict=True, + ) + offerings_client: Final = AsyncHTTPHandler() if offerings_path is not None else None + offerings_manager: Final = ( + ModelOfferingsManager( + path=pathlib.Path(str(offerings_path)), + template=llm_router or Router(model_list=[]), + client=offerings_client, + ) + if offerings_client is not None + else None + ) + if offerings_manager is not None and offerings_client is not None: + if llm_model_list or (llm_router is not None and llm_router.get_model_ids()) or store_model_in_db: + await offerings_client.close() + raise ValueError("External offering mode requires an empty model_list and store_model_in_db disabled") + if not await offerings_manager.reload(initial=True): + await offerings_client.close() + raise ValueError("Initial external offering configuration is invalid") + llm_router = offerings_manager.router + offerings_guard: Final = OfferingAccessGuard(offerings_manager.router) if offerings_manager is not None else None + if offerings_guard is not None: + litellm.logging_callback_manager.add_litellm_callback(offerings_guard) + offerings_task: Final = asyncio.create_task(offerings_manager.run()) if offerings_manager is not None else None + model_info_refresh_disabled: Final = ( "disable_model_info_refresh" in general_settings and general_settings["disable_model_info_refresh"] is True ) model_info_scheduler: Final = ( - None if model_info_refresh_disabled else scheduler if scheduler is not None else AsyncIOScheduler() + None + if model_info_refresh_disabled or offerings_manager is not None + else scheduler + if scheduler is not None + else AsyncIOScheduler() ) if model_info_scheduler is not None: await ProxyStartupEvent.refresh_model_info() @@ -1637,6 +1673,14 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState GracefulShutdownManager.start_shutdown() await GracefulShutdownManager.wait_for_drain() + if offerings_task is not None: + offerings_task.cancel() + await asyncio.gather(offerings_task, return_exceptions=True) + if offerings_guard is not None: + litellm.logging_callback_manager.remove_callback_from_all_lists(offerings_guard, require_self=True) + if offerings_client is not None: + await offerings_client.close() + # Shutdown event - close shared aiohttp session if shared_aiohttp_session is not None: try: @@ -2517,6 +2561,9 @@ app.add_middleware( app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_budget_reservation) app.add_middleware(RedisRequestBatchMiddleware) app.add_middleware(InFlightRequestsMiddleware) +from litellm.proxy.offering_router import OfferingSnapshotMiddleware + +app.add_middleware(OfferingSnapshotMiddleware, router_getter=lambda: llm_router) app.add_middleware(SecurityHeadersMiddleware) app.add_middleware(GZipBufferedResponseMiddleware) @@ -14706,7 +14753,10 @@ def _enrich_model_info_with_litellm_data( }, } gateway_metadata: Final = resolve_gateway_model_metadata( - MappingProxyType({**litellm_model_info, **discovered_model_info}), model_info + MappingProxyType({}) + if llm_router is not None and getattr(llm_router, "model_metadata_authoritative", False) is True + else MappingProxyType({**litellm_model_info, **discovered_model_info}), + model_info, ) published_model: Final = { **model, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 26c410e4827..ff58bd8a9b0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8888,9 +8888,14 @@ def create_model_info_response( deployment_models: Final[tuple[str | None, ...]] = ( listing_info.cost_map_keys if listing_info is not None and listing_info.cost_map_keys else (None,) ) - listed_info: Final = _safe_get_model_info(lookup_model, get_model_info) + authoritative_metadata: Final = ( + llm_router is not None and getattr(llm_router, "model_metadata_authoritative", False) is True + ) + listed_info: Final = None if authoritative_metadata else _safe_get_model_info(lookup_model, get_model_info) candidate_sets: Final = tuple( - _resolve_listing_model_info( + () + if authoritative_metadata + else _resolve_listing_model_info( deployment_model=deployment_model, listed_model=lookup_model, listed_info=listed_info, diff --git a/litellm/router.py b/litellm/router.py index 7c035d1f2a5..c054d80b0ea 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1031,6 +1031,7 @@ class Router: self.routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else [] # Initialize model_group_alias early since it's used in set_model_list + self.model_metadata_authoritative = False self.model_group_alias: dict[str, str | RouterModelGroupAliasItem] = ( model_group_alias or {} ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to gpt-3.5-turbo group @@ -9664,6 +9665,25 @@ class Router: # deployments have been registered. self._finalize_adaptive_router_if_configured() + def snapshot_with_model_list( + self, model_list: Sequence[Mapping[str, object]], *, metadata_authoritative: bool = False + ) -> "Router": + snapshot: Final = copy.copy(self) + snapshot.model_metadata_authoritative = metadata_authoritative + snapshot.pattern_router = PatternMatchRouter() + snapshot.provider_default_deployment_ids = [] + snapshot._zero_cost_cache = {} + snapshot._discovered_model_info_cache = InMemoryCache( + max_size_in_memory=max(len(model_list), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS + ) + snapshot.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + snapshot.get_deployment_model_info + ) + snapshot.set_model_list([dict(model) for model in model_list]) + snapshot.healthy_deployments = snapshot.get_model_list() or [] + _live_routers.add(snapshot) + return snapshot + def _add_deployment(self, deployment: Deployment) -> Deployment: import os diff --git a/litellm/types/proxy/model_inventory.py b/litellm/types/proxy/model_inventory.py new file mode 100644 index 00000000000..b3ea0ae9ddf --- /dev/null +++ b/litellm/types/proxy/model_inventory.py @@ -0,0 +1,15 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Literal + + +@dataclass(frozen=True, slots=True) +class SupplierModelInventory: + models: Mapping[str, Mapping[str, object]] + credential_scope: str | None = None + + +@dataclass(frozen=True, slots=True) +class SupplierInventoryUnavailable: + reason: Literal["authentication", "http", "transport", "malformed"] + credential_scope: str | None = None diff --git a/litellm/types/proxy/model_offerings.py b/litellm/types/proxy/model_offerings.py new file mode 100644 index 00000000000..1694f4e7fc4 --- /dev/null +++ b/litellm/types/proxy/model_offerings.py @@ -0,0 +1,106 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Annotated, Final, Literal, cast # noqa: TID251 # Pydantic runtime schema annotations + +from pydantic import ( + AnyHttpUrl, + BaseModel, + ConfigDict, + Field, + SecretStr, + TypeAdapter, + ValidationInfo, + field_validator, + model_validator, +) + +from litellm.types.proxy.model_metadata import GatewayModelMetadata + + +class OfferingModelInfo(GatewayModelMetadata): + model_config = ConfigDict(extra="forbid", frozen=True) + mode: Literal["chat", "completion", "embedding"] | None = None + + @field_validator("*", mode="before") + @classmethod + def strict_configured_metadata(cls, value: object, info: ValidationInfo) -> object: + if info.field_name is not None: + TypeAdapter[object](cast(object, cls.model_fields[info.field_name].rebuild_annotation())).validate_python( + value, strict=True + ) + return value + + +class SupplierConnection(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["openai", "openrouter", "vercel_ai_gateway", "chatgpt"] + api_base: AnyHttpUrl | None = None + api_key: SecretStr | None = None + headers: Mapping[str, SecretStr] = Field(default_factory=lambda: MappingProxyType({})) + + @field_validator("headers") + @classmethod + def immutable_headers(cls, value: Mapping[str, SecretStr]) -> Mapping[str, SecretStr]: + return MappingProxyType(dict(value)) + + @model_validator(mode="after") + def generic_base_required(self) -> "SupplierConnection": + if self.provider == "openai" and self.api_base is None: + raise ValueError("Generic OpenAI connections require api_base") + if self.provider == "chatgpt" and (self.api_key is not None or self.headers): + raise ValueError( + "ChatGPT connections use the active native OAuth account rather than configured keys or headers" + ) + return self + + +class ModelOffering(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + model_name: Annotated[str, Field(min_length=1, strict=True)] + source: Literal["auto", "manual"] + provider: Annotated[str, Field(min_length=1, strict=True)] + upstream_model: Annotated[str, Field(min_length=1, strict=True)] + enabled: bool = Field(default=True, strict=True) + model_info: OfferingModelInfo = Field(default_factory=OfferingModelInfo) + + @field_validator("model_name", "provider", "upstream_model") + @classmethod + def exact_identifier(cls, value: str) -> str: + if not value.strip() or value != value.strip() or "*" in value: + raise ValueError("Offering identifiers must be nonempty exact names without wildcards") + return value + + +class ModelOfferingsConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + version: Literal[1] + config_poll_seconds: int = Field(default=5, ge=1, le=300, strict=True) + inventory_poll_seconds: int = Field(default=300, ge=1, le=86400, strict=True) + providers: Mapping[str, SupplierConnection] + offerings: tuple[ModelOffering, ...] + + @field_validator("version", mode="before") + @classmethod + def supported_version(cls, value: object) -> Literal[1]: + if isinstance(value, int) and not isinstance(value, bool) and value == 1: + return 1 + raise ValueError("Offering config version must be integer 1") + + @field_validator("providers") + @classmethod + def immutable_providers(cls, value: Mapping[str, SupplierConnection]) -> Mapping[str, SupplierConnection]: + if any(not name.strip() or name != name.strip() or "*" in name for name in value): + raise ValueError("Connection names must be nonempty exact names without wildcards") + return MappingProxyType(dict(value)) + + @model_validator(mode="after") + def known_connections_and_unique_names(self) -> "ModelOfferingsConfig": + names: Final = tuple(offering.model_name for offering in self.offerings) + if len(frozenset(names)) != len(names): + raise ValueError("Each offering must have a unique model_name") + if any(offering.provider not in self.providers for offering in self.offerings): + raise ValueError("Each offering must reference a configured connection") + return self diff --git a/tests/unit/llms/chatgpt/chat/test_transformation.py b/tests/unit/llms/chatgpt/chat/test_transformation.py new file mode 100644 index 00000000000..261936b2656 --- /dev/null +++ b/tests/unit/llms/chatgpt/chat/test_transformation.py @@ -0,0 +1,21 @@ +import json +from pathlib import Path +from typing import Final + +from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig + + +def test_configured_native_endpoint_is_used_with_saved_authorization(tmp_path: Path) -> None: + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text(json.dumps({"access_token": "local-fixture-token", "expires_at": 9999999999})) + config: Final = ChatGPTConfig() + config.authenticator.auth_file = str(auth_file) + with prevent_device_login(): + result: Final = config._get_openai_compatible_provider_info( + model="local-fixture-model", + api_base="https://configured.test/native", + api_key=None, + custom_llm_provider="chatgpt", + ) + assert result == ("https://configured.test/native", "local-fixture-token", "chatgpt") diff --git a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py index a9ced2afcf9..0304bede5a6 100644 --- a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py +++ b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py @@ -68,3 +68,13 @@ class TestChatGPTAuthenticator: assert account_id == "acct-123" mock_write.assert_called_once() assert mock_write.call_args[0][0]["account_id"] == "acct-123" + + +def test_request_scope_never_starts_device_login(tmp_path): + from litellm.llms.chatgpt.authenticator import prevent_device_login + from litellm.llms.chatgpt.common_utils import GetAccessTokenError + + authenticator = Authenticator() + authenticator.auth_file = str(tmp_path / "missing-auth.json") + with prevent_device_login(), pytest.raises(GetAccessTokenError, match="cannot start device login"): + authenticator.get_access_token() diff --git a/tests/unit/llms/openai_like/test_model_info.py b/tests/unit/llms/openai_like/test_model_info.py index 49edd3ea6ec..840e26cb409 100644 --- a/tests/unit/llms/openai_like/test_model_info.py +++ b/tests/unit/llms/openai_like/test_model_info.py @@ -193,3 +193,35 @@ async def test_normalizes_supplier_metadata_without_inventing_input_limit(provid cache=InMemoryCache(), ) assert result == expected + + +async def test_full_inventory_preserves_model_kind_without_fabricating_embedding_output_cap() -> None: + from litellm.llms.openai_like.model_info import get_openai_compatible_model_inventory + from litellm.types.proxy.model_inventory import SupplierModelInventory + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={ + "data": [ + {"id": "embedding", "type": "embedding", "context_window": 4096, "max_tokens": 4096}, + {"id": "language", "type": "language", "max_tokens": 1024}, + ] + }, + ) + ) + ) as client: + handler.client = client + result: Final = await get_openai_compatible_model_inventory( + api_base="https://supplier.test/v1", + provider="vercel_ai_gateway", + headers={}, + client=handler, + cache=InMemoryCache(), + ) + assert isinstance(result, SupplierModelInventory) + assert result.models["embedding"] == {"mode": "embedding", "context_window": 4096} + assert result.models["language"] == {"mode": "chat", "max_output_tokens": 1024} diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py new file mode 100644 index 00000000000..63dfc62d2f3 --- /dev/null +++ b/tests/unit/proxy/test_model_offerings.py @@ -0,0 +1,164 @@ +from pathlib import Path +from typing import Final + +import httpx +import yaml + +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy.model_offerings import ModelOfferingsManager +from litellm.router import Router + + +async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload(tmp_path: Path) -> None: + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "automatic", "source": "auto", "provider": "supplier", "upstream_model": "backend"}, + { + "model_name": "manual", + "source": "manual", + "provider": "supplier", + "upstream_model": "handwritten", + "model_info": {"context_window": 12345}, + }, + ], + } + path.write_text(yaml.safe_dump(config)) + responses: Final = iter( + ( + httpx.Response(200, json={"data": [{"id": "backend", "context_window": 8000}, {"id": "unselected"}]}), + httpx.Response(401, json={"error": "expired"}), + httpx.Response(200, json={"missing": "data"}), + httpx.ReadTimeout("supplier timed out"), + httpx.Response(200, json={"data": [], "error": {"message": "failed"}}), + httpx.Response(200, json={"data": []}), + httpx.Response(200, json={"data": [{"id": "backend", "context_window": 16000}]}), + ) + ) + + def respond(request: httpx.Request) -> httpx.Response: + response: Final = next(responses) + if isinstance(response, Exception): + raise response + return response + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + assert manager.router.model_names == {"automatic", "manual"} + assert manager.router.latest_snapshot().available_models == {"automatic", "manual"} + first: Final = manager.router.latest_snapshot() + for _ in range(4): + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == first.available_models + pinned: Final = manager.router.latest_snapshot() + with manager.router.pin_snapshot(): + assert await manager.reload(force_inventory=True) + assert manager.router.serving_snapshot() is pinned + assert manager.router.latest_snapshot().available_models == {"manual"} + assert manager.router.serving_snapshot().available_models == {"manual"} + assert manager.router.get_model_group_info("automatic") is None + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == {"automatic", "manual"} + assert manager.router.get_model_listing_info("automatic").deployments[0].model_info["context_window"] == 16000 + valid: Final = manager.router.latest_snapshot() + path.write_text("version: 1\nofferings: broken\n") + assert not await manager.reload() + assert manager.router.latest_snapshot() is valid + path.write_text(yaml.safe_dump({**config, "offerings": [config["offerings"][1]]})) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == {"manual"} + assert manager.router.model_names == {"manual"} + + +async def test_new_connection_cannot_reuse_another_credentials_inventory(tmp_path: Path) -> None: + path: Final = tmp_path / "offerings.yaml" + first_config: Final = { + "version": 1, + "providers": { + "supplier": {"provider": "openai", "api_base": "https://supplier.test/v1", "api_key": "first-private-key"} + }, + "offerings": [ + {"model_name": "selected", "source": "auto", "provider": "supplier", "upstream_model": "backend"} + ], + } + path.write_text(yaml.safe_dump(first_config)) + responses: Final = iter( + ( + httpx.Response(200, json={"data": [{"id": "backend"}]}), + httpx.Response(401), + httpx.Response(200, json={"data": [{"id": "backend"}]}), + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + changed: Final = { + **first_config, + "providers": {"supplier": {**first_config["providers"]["supplier"], "api_key": "second-private-key"}}, + } + path.write_text(yaml.safe_dump(changed)) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == set() + assert manager.router.get_model_group_info("selected") is None + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + + +async def test_pinned_metadata_does_not_read_new_snapshot_shared_catalog(tmp_path: Path) -> None: + from litellm.proxy.utils import create_model_info_response + + path: Final = tmp_path / "offerings.yaml" + initial: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + { + "model_name": "snapshot-only-model", + "source": "manual", + "provider": "supplier", + "upstream_model": "snapshot-only-backend", + "model_info": {"context_window": 8000}, + } + ], + } + path.write_text(yaml.safe_dump(initial)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + with manager.router.pin_snapshot(): + before: Final = create_model_info_response("snapshot-only-model", "openai", llm_router=manager.router) + changed: Final = { + **initial, + "offerings": [ + {**initial["offerings"][0], "model_info": {"context_window": 16000, "max_output_tokens": 5001}} + ], + } + path.write_text(yaml.safe_dump(changed)) + assert await manager.reload() + assert create_model_info_response("snapshot-only-model", "openai", llm_router=manager.router) == before + after: Final = create_model_info_response("snapshot-only-model", "openai", llm_router=manager.router) + assert before["context_window"] == 8000 + assert "max_output_tokens" not in before + assert after["context_window"] == 16000 + assert after["max_output_tokens"] == 5001 diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py new file mode 100644 index 00000000000..87f3509d569 --- /dev/null +++ b/tests/unit/proxy/test_offering_router.py @@ -0,0 +1,45 @@ +from types import MappingProxyType +from typing import Final + +import litellm +from litellm.caching.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.offering_router import OfferingAccessGuard, OfferingRouterView, OfferingServingSnapshot +from litellm.router import Router +from litellm.types.utils import CallTypes + + +async def test_unavailable_and_unselected_offerings_cannot_forward_or_override_connection() -> None: + router: Final = OfferingRouterView( + OfferingServingSnapshot( + Router(model_list=[]), + frozenset({"selected"}), + MappingProxyType({"missing": "absent from supplier"}), + frozenset({"openai/backend"}), + ) + ) + guard: Final = OfferingAccessGuard(router) + auth: Final = UserAPIKeyAuth() + cache: Final = DualCache() + assert await guard.async_filter_listed_models(auth, ("selected", "missing", "unselected")) == ("selected",) + assert await guard.async_pre_call_hook(auth, cache, {"model": "selected"}, "acompletion") is None + assert isinstance( + await guard.async_pre_call_hook(auth, cache, {"model": "missing"}, "acompletion"), + litellm.ServiceUnavailableError, + ) + assert isinstance( + await guard.async_pre_call_hook(auth, cache, {"model": "unselected"}, "acompletion"), litellm.NotFoundError + ) + assert isinstance( + await guard.async_pre_call_hook( + auth, cache, {"model": "selected", "api_base": "https://other.test"}, "acompletion" + ), + litellm.BadRequestError, + ) + await guard.async_pre_call_deployment_hook({"model": "openai/backend"}, CallTypes.acompletion) + try: + await guard.async_pre_call_deployment_hook({"model": "openai/unselected"}, CallTypes.acompletion) + except litellm.NotFoundError: + pass + else: + raise AssertionError("An unselected concrete deployment was allowed") diff --git a/tests/unit/types/proxy/test_model_offerings.py b/tests/unit/types/proxy/test_model_offerings.py new file mode 100644 index 00000000000..11270c495d1 --- /dev/null +++ b/tests/unit/types/proxy/test_model_offerings.py @@ -0,0 +1,42 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.types.proxy.model_offerings import ModelOfferingsConfig + + +@pytest.mark.parametrize( + "change", + ( + {"version": True}, + {"config_poll_seconds": 0}, + {"inventory_poll_seconds": "300"}, + { + "offerings": [ + {"model_name": "duplicate", "source": "manual", "provider": "supplier", "upstream_model": "backend"} + ] + * 2 + }, + { + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "backend", + "model_info": {"context_window": -1}, + } + ] + }, + {"providers": {"supplier": {"provider": "chatgpt", "api_key": "ignored-key"}}}, + ), +) +def test_invalid_operator_edits_are_rejected(change) -> None: + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [], + } + with pytest.raises(ValidationError): + ModelOfferingsConfig.model_validate({**config, **change}) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5c1107073a3..1bfb51d49dd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30045,6 +30045,11 @@ export interface components { * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called. */ model_list_healthy_only?: boolean | null; + /** + * Model Offerings Path + * @description Path to the authoritative external offering file, reloaded without restarting the proxy + */ + model_offerings_path?: string | null; /** * Otel * @description [BETA] OpenTelemetry support - this might change, use with caution. From c3cb22810b23ee8f46f1c8e8cb4879e367d13c18 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 5 Oct 2026 17:04:58 +0200 Subject: [PATCH 04/34] fix(proxy): isolate offering snapshot dispatch and validate supplier metadata --- examples/model_offerings/README.md | 4 +- litellm/llms/chatgpt/model_info.py | 24 ++++- litellm/llms/openai_like/model_info.py | 28 +++++- litellm/proxy/model_offerings.py | 35 +++++++- litellm/proxy/offering_router.py | 14 ++- litellm/router.py | 6 ++ litellm/types/proxy/model_metadata.py | 8 +- litellm/types/proxy/model_offerings.py | 7 +- tests/unit/proxy/test_model_offerings.py | 110 ++++++++++++++++++++++- tests/unit/proxy/test_offering_router.py | 15 +++- 10 files changed, 232 insertions(+), 19 deletions(-) diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index db8cbff25a9..de4c756be2f 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -53,10 +53,10 @@ offerings: These example token values illustrate operator configuration, not a claim about the example supplier models. Request defaults express chosen budgets and do not populate independent output capability limits -`source` belongs to each offering. AUTO entries require membership in the latest successful full inventory for their connection. Missing entries disappear from model listings and return a useful unavailable error before forwarding. A returning entry becomes available again. MANUAL entries use only handwritten definitions and are independent of supplier inventory membership. Both sources accept shared `model_info` overrides; explicit overrides take precedence over downloaded metadata. Wildcards and duplicate public aliases are rejected +`source` belongs to each offering. AUTO entries require membership in the latest successful full inventory for their connection. Missing entries disappear from model listings and return a useful unavailable error before forwarding. A returning entry becomes available again. AUTO metadata combines the verified catalog with downloaded supplier fields and then explicit overrides, in increasing priority, and freezes the result in the serving snapshot. MANUAL entries use only handwritten definitions and are independent of supplier inventory membership. Both sources accept shared `model_info` overrides. Wildcards and duplicate public aliases are rejected Startup validates the complete file and awaits initial inventory requests. AUTO entries remain unavailable when no successful inventory exists; MANUAL entries remain usable. A successful empty inventory means absence. HTTP errors, authentication failures, timeouts and malformed responses retain the last successful inventory for unchanged credentials. Changing connection credentials starts with unknown availability and forces a fresh inventory. ChatGPT uses its existing native OAuth account and cannot initiate device login from discovery or request handling -The file is checked every five seconds by default, with a configurable range of 1 to 300 seconds. Inventories refresh every 300 seconds by default, with a range of 1 to 86400 seconds. Invalid file edits retain the whole last-valid serving configuration, including credentials, and log only the error kind. Unselected inventory entries never become public routes. Caller connection overrides and runtime model-management mutations are rejected in external mode; edit the external file instead +The file is checked every five seconds by default, with a configurable range of 1 to 300 seconds. Inventories refresh every 300 seconds by default, with a range of 1 to 86400 seconds. Changing that interval triggers a fresh inventory request and resets its deadline. Invalid file edits retain the whole last-valid serving configuration, including credentials, and log only the error kind. Unselected inventory entries never become public routes. Caller connection overrides and runtime model-management mutations are rejected in external mode; edit the external file instead Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The inventory HTTP client and polling task are closed at proxy shutdown after requests drain diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index 46e306a7e25..727fad68349 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -6,7 +6,7 @@ from types import MappingProxyType from typing import Final import httpx -from pydantic import BaseModel, ConfigDict, ValidationError, field_validator +from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -34,6 +34,22 @@ class _ChatGPTModel(GatewayModelMetadata): supports_parallel_tool_calls: bool | None = None multi_agent_reasoning_effort: str | None = None + @field_validator("*", mode="before") + @classmethod + def strict_supplier_metadata(cls, value: object, info: ValidationInfo) -> object: + normalized: Final = ( + None + if info.field_name in ("context_window", "max_input_tokens", "max_output_tokens", "max_context_window") + and type(value) is int + and value == 0 + else value + ) + if info.field_name in GatewayModelMetadata.model_fields or info.field_name == "max_context_window": + cls.validate_field_input( + "context_window" if info.field_name == "max_context_window" else info.field_name, normalized + ) + return normalized + @field_validator("max_context_window", mode="before") @classmethod def maximum_context(cls, value: object) -> int | None: @@ -167,7 +183,11 @@ async def get_chatgpt_model_inventory( response.raise_for_status() catalog: Final = _ChatGPTCatalog.model_validate_json(response.content) ids: Final = tuple(card.slug for card in catalog.models) - if catalog.error is not None or any(not model_id for model_id in ids) or len(frozenset(ids)) != len(ids): + if ( + catalog.error is not None + or any(not model_id.strip() or model_id != model_id.strip() for model_id in ids) + or len(frozenset(ids)) != len(ids) + ): return SupplierInventoryUnavailable("malformed", credential_scope) inventory: Final = SupplierModelInventory( MappingProxyType({card.slug: card.metadata() for card in catalog.models}), credential_scope diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 70966605204..de563c3885c 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -5,7 +5,7 @@ from types import MappingProxyType from typing import Annotated, Final, TypeAlias import httpx -from pydantic import BaseModel, BeforeValidator, ConfigDict, ValidationError +from pydantic import BaseModel, BeforeValidator, ConfigDict, ValidationError, ValidationInfo, field_validator from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -22,7 +22,11 @@ _EMPTY_LIMITS: Final[Mapping[str, object]] = MappingProxyType({}) def _positive_limit(value: object) -> int | None: - return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + if value is None or type(value) is int and value == 0: + return None + if type(value) is int and value > 0: + return value + raise ValueError("Supplier token limits must be nonnegative integers or null") _TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] @@ -67,6 +71,20 @@ class _ModelCard(GatewayModelMetadata): reasoning: _Reasoning | None = None reasoning_options: tuple[_ReasoningOption, ...] = () + @field_validator("*", mode="before") + @classmethod + def strict_supplier_metadata(cls, value: object, info: ValidationInfo) -> object: + normalized: Final = ( + None + if info.field_name in ("context_window", "max_input_tokens", "max_output_tokens") + and type(value) is int + and value == 0 + else value + ) + if info.field_name in GatewayModelMetadata.model_fields: + cls.validate_field_input(info.field_name, normalized) + return normalized + def token_limits(self, provider: str) -> Mapping[str, object]: context: Final = self.context_window or self.max_model_len or self.context_length parameters: Final = self.supported_parameters @@ -198,7 +216,11 @@ async def get_openai_compatible_model_inventory( response.raise_for_status() models: Final = _ModelList.model_validate_json(response.content) ids: Final = tuple(card.id for card in models.data) - if models.error is not None or any(not model_id for model_id in ids) or len(frozenset(ids)) != len(ids): + if ( + models.error is not None + or any(not model_id.strip() or model_id != model_id.strip() for model_id in ids) + or len(frozenset(ids)) != len(ids) + ): unavailable: Final = SupplierInventoryUnavailable("malformed") cache.set_cache(cache_key, unavailable, ttl=60) return unavailable diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index ddf4c5361db..b9616f5385a 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -13,6 +13,7 @@ import yaml from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.llms.chatgpt.authenticator import Authenticator, prevent_device_login from litellm.llms.chatgpt.model_info import get_chatgpt_model_inventory from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -20,6 +21,7 @@ from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY, from litellm.proxy.offering_router import OfferingRouterView, OfferingServingSnapshot from litellm.router import Router from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.types.proxy.model_offerings import ModelOffering, ModelOfferingsConfig, SupplierConnection @@ -151,7 +153,23 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, if offering.source == "auto" and state.inventory is not None else {} ) - metadata: Final = {**supplier_info, **offering.model_info.model_dump(mode="json", exclude_none=True)} + catalog: Final = GetModelCostMap.loaded_model_cost_map() + bundled_info: Final = ( + catalog.get(f"{connection.provider}/{offering.upstream_model}") + or (catalog.get(offering.upstream_model, {}) if connection.provider == "openai" else {}) + if offering.source == "auto" + else {} + ) + bundled_metadata: Final = GatewayModelMetadata.model_validate(bundled_info).model_dump( + mode="json", exclude_none=True + ) + bundled_mode: Final = bundled_info.get("mode") + metadata: Final = { + **bundled_metadata, + **({"mode": bundled_mode} if bundled_mode in ("chat", "completion", "embedding") else {}), + **supplier_info, + **offering.model_info.model_dump(mode="json", exclude_none=True), + } deployment_id: Final = ( "offering-" + hashlib.sha256( @@ -269,7 +287,9 @@ class ModelOfferingsManager: for offering in loaded.config.offerings if offering.model_name in available ) - snapshot: Final = OfferingServingSnapshot(candidate, available, unavailable, allowed_deployments) + snapshot: Final = OfferingServingSnapshot( + candidate, available, unavailable, allowed_deployments, frozenset(candidate.get_model_ids()) + ) self.config = loaded.config self.config_fingerprint = loaded.fingerprint self.providers = providers @@ -313,7 +333,16 @@ class ModelOfferingsManager: name not in self.providers or self.providers[name].connection != state.connection for name, state in providers.items() ) - refresh_due: Final = initial or force_inventory or connection_changed or now >= self.next_inventory_refresh + interval_changed: Final = ( + self.config is not None and loaded.config.inventory_poll_seconds != self.config.inventory_poll_seconds + ) + refresh_due: Final = ( + initial + or force_inventory + or connection_changed + or interval_changed + or now >= self.next_inventory_refresh + ) refreshed: Final = await self._refresh_providers(providers) if refresh_due else providers if not self._apply(loaded, refreshed): return False diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index ae46285177b..587aaa2493e 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -4,6 +4,7 @@ from contextvars import ContextVar from dataclasses import dataclass from typing import Final, cast # noqa: TID251 # native Router dynamic attribute dispatch +from pydantic import TypeAdapter from starlette.types import ASGIApp, Receive, Scope, Send import litellm @@ -21,6 +22,7 @@ class OfferingServingSnapshot: available_models: frozenset[str] unavailable_models: Mapping[str, str] allowed_deployments: frozenset[str] + allowed_deployment_ids: frozenset[str] = frozenset() _PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] = ContextVar( @@ -28,6 +30,7 @@ _PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] ) _MODEL_MUTATIONS: Final = frozenset(("set_model_list", "add_deployment", "upsert_deployment", "delete_deployment")) _VIEW_MEMBERS: Final = frozenset(("publish_snapshot", "pin_snapshot", "serving_snapshot", "latest_snapshot")) +_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) class OfferingRouterView(Router): @@ -153,7 +156,16 @@ class OfferingAccessGuard(CustomLogger): CallTypes.embedding, ): return - if model not in self.router.serving_snapshot().allowed_deployments: + metadata: Final = _METADATA_ADAPTER.validate_python( + kwargs.get("litellm_metadata") or kwargs.get("metadata") or {} + ) + raw_model_info: Final = metadata.get("model_info") + model_info: Final = ( + _METADATA_ADAPTER.validate_python(raw_model_info) if isinstance(raw_model_info, Mapping) else {} + ) + deployment_id: Final = model_info.get("id") + snapshot: Final = self.router.serving_snapshot() + if model not in snapshot.allowed_deployments or deployment_id not in snapshot.allowed_deployment_ids: raise litellm.NotFoundError( message="The requested deployment is not an enabled gateway offering", model=model, llm_provider="" ) diff --git a/litellm/router.py b/litellm/router.py index c054d80b0ea..ec0f8c9c37a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9681,6 +9681,12 @@ class Router: ) snapshot.set_model_list([dict(model) for model in model_list]) snapshot.healthy_deployments = snapshot.get_model_list() or [] + # Factory closures and the OpenAI-compatible wrapper must dispatch through + # this snapshot, rather than the template whose attributes were copied. + # Rebinding endpoints does not register callbacks or recreate clients. + snapshot.initialize_assistants_endpoint() + snapshot.initialize_router_endpoints() + snapshot.chat = litellm.Chat(params=snapshot.default_litellm_params, router_obj=snapshot) _live_routers.add(snapshot) return snapshot diff --git a/litellm/types/proxy/model_metadata.py b/litellm/types/proxy/model_metadata.py index 8fe1639ca74..69c21a52ac4 100644 --- a/litellm/types/proxy/model_metadata.py +++ b/litellm/types/proxy/model_metadata.py @@ -1,6 +1,6 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Annotated, Final +from typing import Annotated, Final, cast # noqa: TID251 # Pydantic runtime schema boundary from pydantic import ( BaseModel, @@ -64,6 +64,12 @@ class GatewayModelMetadata(BaseModel): default=None, description="Configured request defaults, independent of supplier capability limits" ) + @classmethod + def validate_field_input(cls, field_name: str, value: object) -> None: + TypeAdapter[object]( + cast(object, cls.model_fields[field_name].rebuild_annotation()) # cast-ok: runtime Pydantic field schema + ).validate_python(value, strict=True) + @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") @classmethod def positive_limit(cls, value: object) -> int | None: diff --git a/litellm/types/proxy/model_offerings.py b/litellm/types/proxy/model_offerings.py index 1694f4e7fc4..87aba7b80f4 100644 --- a/litellm/types/proxy/model_offerings.py +++ b/litellm/types/proxy/model_offerings.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Annotated, Final, Literal, cast # noqa: TID251 # Pydantic runtime schema annotations +from typing import Annotated, Final, Literal from pydantic import ( AnyHttpUrl, @@ -8,7 +8,6 @@ from pydantic import ( ConfigDict, Field, SecretStr, - TypeAdapter, ValidationInfo, field_validator, model_validator, @@ -25,9 +24,7 @@ class OfferingModelInfo(GatewayModelMetadata): @classmethod def strict_configured_metadata(cls, value: object, info: ValidationInfo) -> object: if info.field_name is not None: - TypeAdapter[object](cast(object, cls.model_fields[info.field_name].rebuild_annotation())).validate_python( - value, strict=True - ) + cls.validate_field_input(info.field_name, value) return value diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 63dfc62d2f3..2f399859992 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -33,6 +33,10 @@ async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload httpx.Response(200, json={"missing": "data"}), httpx.ReadTimeout("supplier timed out"), httpx.Response(200, json={"data": [], "error": {"message": "failed"}}), + httpx.Response(200, json={"data": [{"id": "backend", "context_window": "bad"}]}), + httpx.Response(200, json={"data": [{"id": "backend", "supports_function_calling": "bad"}]}), + httpx.Response(200, json={"data": [{"id": "backend", "supported_endpoints": 123}]}), + httpx.Response(200, json={"data": [{"id": " backend "}]}), httpx.Response(200, json={"data": []}), httpx.Response(200, json={"data": [{"id": "backend", "context_window": 16000}]}), ) @@ -55,9 +59,12 @@ async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload assert manager.router.model_names == {"automatic", "manual"} assert manager.router.latest_snapshot().available_models == {"automatic", "manual"} first: Final = manager.router.latest_snapshot() - for _ in range(4): + for _ in range(8): assert await manager.reload(force_inventory=True) assert manager.router.latest_snapshot().available_models == first.available_models + assert ( + manager.router.get_model_listing_info("automatic").deployments[0].model_info["context_window"] == 8000 + ) pinned: Final = manager.router.latest_snapshot() with manager.router.pin_snapshot(): assert await manager.reload(force_inventory=True) @@ -162,3 +169,104 @@ async def test_pinned_metadata_does_not_read_new_snapshot_shared_catalog(tmp_pat assert "max_output_tokens" not in before assert after["context_window"] == 16000 assert after["max_output_tokens"] == 5001 + + +async def test_snapshot_factory_dispatch_and_reload_resources_are_isolated() -> None: + import gc + import weakref + + import litellm + + template: Final = Router(model_list=[], num_retries=0) + callback_counts: Final = tuple( + len(callbacks) + for callbacks in ( + litellm.success_callback, + litellm.failure_callback, + litellm._async_success_callback, + litellm._async_failure_callback, + ) + ) + models: Final = ( + {"model_name": "selected", "litellm_params": {"model": "openai/fixture", "api_key": "local-only"}}, + ) + first: Final = template.snapshot_with_model_list(models) + response: Final = await first.aresponses(model="selected", input="fixture", mock_response="local-only") + assert response.output[0].content[0].text == "local-only" + assert template.get_model_names() == [] + references: Final = tuple(weakref.ref(template.snapshot_with_model_list(models)) for _ in range(30)) + gc.collect() + assert all(reference() is None for reference in references) + assert callback_counts == tuple( + len(callbacks) + for callbacks in ( + litellm.success_callback, + litellm.failure_callback, + litellm._async_success_callback, + litellm._async_failure_callback, + ) + ) + + +async def test_auto_catalog_fallback_is_frozen_and_manual_metadata_stays_handwritten(tmp_path: Path) -> None: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + path: Final = tmp_path / "offerings.yaml" + upstream: Final = "text-embedding-3-large" + bundled: Final = GetModelCostMap.loaded_model_cost_map()[upstream] + assert bundled["max_input_tokens"] > 0 + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "automatic", "source": "auto", "provider": "supplier", "upstream_model": upstream}, + {"model_name": "manual", "source": "manual", "provider": "supplier", "upstream_model": upstream}, + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"data": [{"id": upstream, "context_window": 12345}]}) + ) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) + assert await manager.reload(initial=True) + auto: Final = manager.router.get_model_listing_info("automatic").deployments[0].model_info + manual: Final = manager.router.get_model_listing_info("manual").deployments[0].model_info + assert auto["context_window"] == 12345 + assert auto["max_input_tokens"] == bundled["max_input_tokens"] + assert auto["mode"] == "embedding" + assert manual.get("context_window") is None + assert manual.get("max_input_tokens") is None + + +async def test_changed_inventory_interval_refreshes_without_retaining_old_deadline(tmp_path: Path) -> None: + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "inventory_poll_seconds": 86400, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "automatic", "source": "auto", "provider": "supplier", "upstream_model": "backend"} + ], + } + path.write_text(yaml.safe_dump(config)) + responses: Final = iter( + (httpx.Response(200, json={"data": [{"id": "backend"}]}), httpx.Response(200, json={"data": []})) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + assert manager.router.latest_snapshot().available_models == {"automatic"} + path.write_text(yaml.safe_dump({**config, "inventory_poll_seconds": 1})) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == set() + assert manager.next_inventory_refresh == 1.0 diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 87f3509d569..975bbe58477 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -16,6 +16,7 @@ async def test_unavailable_and_unselected_offerings_cannot_forward_or_override_c frozenset({"selected"}), MappingProxyType({"missing": "absent from supplier"}), frozenset({"openai/backend"}), + frozenset({"selected-deployment"}), ) ) guard: Final = OfferingAccessGuard(router) @@ -36,10 +37,22 @@ async def test_unavailable_and_unselected_offerings_cannot_forward_or_override_c ), litellm.BadRequestError, ) - await guard.async_pre_call_deployment_hook({"model": "openai/backend"}, CallTypes.acompletion) + await guard.async_pre_call_deployment_hook( + {"model": "openai/backend", "metadata": {"model_info": {"id": "selected-deployment"}}}, + CallTypes.acompletion, + ) try: await guard.async_pre_call_deployment_hook({"model": "openai/unselected"}, CallTypes.acompletion) except litellm.NotFoundError: pass else: raise AssertionError("An unselected concrete deployment was allowed") + try: + await guard.async_pre_call_deployment_hook( + {"model": "openai/backend", "metadata": {"model_info": {"id": "unavailable-other-connection"}}}, + CallTypes.acompletion, + ) + except litellm.NotFoundError: + pass + else: + raise AssertionError("An unavailable deployment sharing the backend model was allowed") From b5de81cdc7f93a639b401e1a0a23722c47407418 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Tue, 6 Oct 2026 08:30:36 +0200 Subject: [PATCH 05/34] fix(proxy): prevent duplicate offering Authorization headers --- litellm/proxy/model_offerings.py | 11 +++- tests/unit/proxy/test_model_offerings.py | 65 ++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index b9616f5385a..2fe89e7f859 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -183,7 +183,16 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, "model": f"{connection.provider}/{offering.upstream_model}", **({"api_base": connection.api_base} if connection.api_base is not None else {}), **({"api_key": connection.api_key} if connection.api_key is not None else {}), - **({"extra_headers": dict(connection.headers)} if connection.headers else {}), + **( + { + "extra_headers": { + "Authorization" if name == "authorization" else name: value + for name, value in connection.headers.items() + } + } + if connection.headers + else {} + ), }, "model_info": {**metadata, "id": deployment_id, "blocked": reason is not None}, } diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 2f399859992..204052e4188 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -2,6 +2,8 @@ from pathlib import Path from typing import Final import httpx +import pytest +from openai import AsyncOpenAI import yaml from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -270,3 +272,66 @@ async def test_changed_inventory_interval_refreshes_without_retaining_old_deadli assert await manager.reload() assert manager.router.latest_snapshot().available_models == set() assert manager.next_inventory_refresh == 1.0 + + +@pytest.mark.parametrize("authorization_name", (None, "authorization", "AUTHORIZATION")) +async def test_native_client_forwards_one_supplier_authorization_value( + tmp_path: Path, authorization_name: str | None +) -> None: + path: Final = tmp_path / "offerings.yaml" + api_key: Final = "supplier-fixture-key.with-symbols=" + expected_authorization: Final = ( + "Bearer explicit-fixture-override" if authorization_name is not None else f"Bearer {api_key}" + ) + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "supplier": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "api_key": api_key, + "headers": ( + {authorization_name: expected_authorization} if authorization_name is not None else {} + ), + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "fixture-backend", + } + ], + } + ) + ) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers.get_list("authorization") == [expected_authorization] + return httpx.Response( + 200, + json={ + "id": "fixture-response", + "object": "chat.completion", + "created": 1, + "model": "fixture-backend", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "accepted"}, "finish_reason": "stop"} + ], + }, + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler.client = http_client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[], num_retries=0), client=handler) + assert await manager.reload(initial=True) + async with AsyncOpenAI(api_key=api_key, http_client=http_client, max_retries=0) as sdk_client: + response: Final = await manager.router.acompletion( + model="selected", messages=[{"role": "user", "content": "authenticate"}], client=sdk_client + ) + assert response.choices[0].message.content == "accepted" From 5b1b3f107b89d161b03f29e263076b51939acff1 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 10:43:27 +0200 Subject: [PATCH 06/34] test(proxy): preserve customer permissions for external offerings --- tests/unit/proxy/test_offering_router.py | 37 +++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 975bbe58477..b2b38732890 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -2,8 +2,10 @@ from types import MappingProxyType from typing import Final import litellm +import pytest from litellm.caching.caching import DualCache -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_EndUserTable, ModelAccessDeniedProxyException, ProxyErrorTypes, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import can_customer_access_model, can_key_call_model from litellm.proxy.offering_router import OfferingAccessGuard, OfferingRouterView, OfferingServingSnapshot from litellm.router import Router from litellm.types.utils import CallTypes @@ -56,3 +58,36 @@ async def test_unavailable_and_unselected_offerings_cannot_forward_or_override_c pass else: raise AssertionError("An unavailable deployment sharing the backend model was allowed") + + +async def test_globally_available_offerings_preserve_customer_and_key_model_permissions() -> None: + native: Final = Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": f"openai/{name}", "api_key": "fixture-key"}} + for name in ("allowed", "restricted") + ] + ) + router: Final = OfferingRouterView( + OfferingServingSnapshot( + native, + frozenset({"allowed", "restricted"}), + MappingProxyType({}), + frozenset({"openai/allowed", "openai/restricted"}), + ) + ) + guard: Final = OfferingAccessGuard(router) + key: Final = UserAPIKeyAuth(models=["allowed", "restricted"]) + customer: Final = LiteLLM_EndUserTable(user_id="fixture-customer", blocked=False, models=["allowed"]) + with router.pin_snapshot(): + assert await guard.async_pre_call_hook(key, DualCache(), {"model": "restricted"}, "acompletion") is None + assert await can_key_call_model("restricted", None, key, router) is True + with pytest.raises(ModelAccessDeniedProxyException) as denied_customer: + can_customer_access_model("restricted", customer, router, key) + assert denied_customer.value.code == "403" + assert denied_customer.value.type == ProxyErrorTypes.customer_model_access_denied + assert can_customer_access_model("allowed", customer, router, key) is True + assert await can_key_call_model("allowed", None, key, router) is True + with pytest.raises(ModelAccessDeniedProxyException) as denied_key: + await can_key_call_model("restricted", None, UserAPIKeyAuth(models=["allowed"]), router) + assert denied_key.value.code == "403" + assert denied_key.value.type == ProxyErrorTypes.key_model_access_denied From 37b2cef23709c3010571861f1136cd6f02a8c121 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 10:49:55 +0200 Subject: [PATCH 07/34] fix(proxy): preserve offering availability through model aliases --- litellm/proxy/offering_router.py | 39 +++++++++++++--- litellm/proxy/proxy_server.py | 4 +- tests/unit/proxy/test_offering_router.py | 57 +++++++++++++++++++++++- 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index 587aaa2493e..c7e885eb6c2 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -11,8 +11,10 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.llms.chatgpt.authenticator import prevent_device_login -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator, alias_target, caller_alias_maps from litellm.router import Router +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.utils import CallTypes, CallTypesLiteral @@ -95,14 +97,40 @@ class OfferingSnapshotMiddleware: class OfferingAccessGuard(CustomLogger): - def __init__(self, router: OfferingRouterView) -> None: + def __init__(self, router: OfferingRouterView, general_settings: Mapping[str, object] | None = None) -> None: self.router = router + self.general_settings = general_settings if general_settings is not None else {} + + def _offering_name(self, name: str, auth: UserAPIKeyAuth) -> str: + snapshot: Final = self.router.serving_snapshot() + caller_target: Final = ( + alias_target( + name, + caller_alias_maps(auth.aliases, auth.team_model_aliases, auth.team_id, auth.team_id), + snapshot.available_models | frozenset(snapshot.unavailable_models), + ) + or name + ) + target: Final = resolve_model_group_alias(snapshot.router.model_group_alias, caller_target) or caller_target + names: Final = [ + row["model_name"] + for row in (snapshot.router.get_model_list() or []) + if isinstance(row.get("model_name"), str) + and ( + auth.user_role == LitellmUserRoles.PROXY_ADMIN + or not isinstance(row.get("model_info"), Mapping) + or row["model_info"].get("team_id") in (None, auth.team_id) + ) + ] + return TeamModelNameTranslator.resolve_public_name(target, names, snapshot.router, self.general_settings) async def async_filter_listed_models( self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] ) -> Sequence[str]: snapshot: Final = self.router.serving_snapshot() - return tuple(name for name in model_names if name in snapshot.available_models) + return tuple( + name for name in model_names if self._offering_name(name, user_api_key_dict) in snapshot.available_models + ) async def async_pre_call_hook( self, @@ -133,9 +161,10 @@ class OfferingAccessGuard(CustomLogger): if not isinstance(model, str): return None snapshot: Final = self.router.serving_snapshot() - if model in snapshot.available_models: + offering: Final = self._offering_name(model, user_api_key_dict) + if offering in snapshot.available_models: return None - reason: Final = snapshot.unavailable_models.get(model) + reason: Final = snapshot.unavailable_models.get(offering) if reason is not None: return litellm.ServiceUnavailableError( message=f"Model '{model}' is unavailable: {reason}", model=model, llm_provider="" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ad8b2045faf..c890646975d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1616,7 +1616,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState await offerings_client.close() raise ValueError("Initial external offering configuration is invalid") llm_router = offerings_manager.router - offerings_guard: Final = OfferingAccessGuard(offerings_manager.router) if offerings_manager is not None else None + offerings_guard: Final = ( + OfferingAccessGuard(offerings_manager.router, general_settings) if offerings_manager is not None else None + ) if offerings_guard is not None: litellm.logging_callback_manager.add_litellm_callback(offerings_guard) offerings_task: Final = asyncio.create_task(offerings_manager.run()) if offerings_manager is not None else None diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index b2b38732890..6b21891f7c7 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -4,7 +4,13 @@ from typing import Final import litellm import pytest from litellm.caching.caching import DualCache -from litellm.proxy._types import LiteLLM_EndUserTable, ModelAccessDeniedProxyException, ProxyErrorTypes, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_EndUserTable, + LitellmUserRoles, + ModelAccessDeniedProxyException, + ProxyErrorTypes, + UserAPIKeyAuth, +) from litellm.proxy.auth.auth_checks import can_customer_access_model, can_key_call_model from litellm.proxy.offering_router import OfferingAccessGuard, OfferingRouterView, OfferingServingSnapshot from litellm.router import Router @@ -91,3 +97,52 @@ async def test_globally_available_offerings_preserve_customer_and_key_model_perm await can_key_call_model("restricted", None, UserAPIKeyAuth(models=["allowed"]), router) assert denied_key.value.code == "403" assert denied_key.value.type == ProxyErrorTypes.key_model_access_denied + + +async def test_public_and_router_aliases_follow_offering_availability_without_crossing_teams() -> None: + active: Final = "model_name_team1_active" + missing: Final = "model_name_team1_missing" + other_team: Final = "model_name_team2_active" + native: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"openai/{name}", "api_key": "fixture-key"}, + "model_info": {"team_id": team, "team_public_model_name": public}, + } + for name, team, public in ( + (active, "team1", "fast"), + (missing, "team1", "gone"), + (other_team, "team2", "gone"), + ) + ], + model_group_alias={"active-alias": active, "missing-alias": missing}, + ) + view: Final = OfferingRouterView( + OfferingServingSnapshot( + native, frozenset({active, other_team}), MappingProxyType({missing: "absent"}), frozenset() + ) + ) + auth: Final = UserAPIKeyAuth(team_id="team1", aliases={"my-fast": active, "my-gone": missing}) + guard: Final = OfferingAccessGuard(view) + with view.pin_snapshot(): + assert await guard.async_filter_listed_models( + auth, ("fast", "gone", "active-alias", "missing-alias", "my-fast", "my-gone") + ) == ( + "fast", + "active-alias", + "my-fast", + ) + assert await guard.async_pre_call_hook(auth, DualCache(), {"model": "fast"}, "acompletion") is None + assert isinstance( + await guard.async_pre_call_hook(auth, DualCache(), {"model": "gone"}, "acompletion"), + litellm.ServiceUnavailableError, + ) + assert isinstance( + await guard.async_pre_call_hook(auth, DualCache(), {"model": "missing-alias"}, "acompletion"), + litellm.ServiceUnavailableError, + ) + admin: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + assert await guard.async_filter_listed_models(admin, ("fast",)) == ("fast",) + legacy: Final = OfferingAccessGuard(view, {"use_team_public_model_name": False}) + assert await legacy.async_filter_listed_models(auth, ("fast", active)) == (active,) From 970477081ca0385b3d23893650d598e6928f6bac Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 11:43:50 +0200 Subject: [PATCH 08/34] fix(proxy): enforce offering boundaries and preserve snapshot affinity --- ci_cd/generate_model_prices_schema.py | 11 +- litellm/llms/chatgpt/model_info.py | 19 ++- litellm/llms/model_inventory.py | 110 ++++++++++++++ .../common_utils/encrypt_decrypt_utils.py | 17 +++ litellm/proxy/model_offerings.py | 73 +++++---- litellm/proxy/offering_router.py | 18 ++- litellm/router.py | 42 +----- .../router_utils/model_request_defaults.py | 11 +- model_prices_and_context_window.schema.json | 10 +- tests/unit/llms/chatgpt/test_model_info.py | 29 ++++ tests/unit/llms/test_model_inventory.py | 31 ++++ .../test_encrypt_decrypt_utils.py | 18 +++ .../test_team_model_name_translation.py | 6 + tests/unit/proxy/test_model_offerings.py | 142 +++++++++++++++++- tests/unit/proxy/test_offering_router.py | 26 +++- .../test_model_request_defaults.py | 11 ++ tests/unit/test_internal_context.py | 1 + tests/unit/test_model_prices_schema.py | 9 +- .../unit/test_router_model_cost_isolation.py | 55 +++++++ 19 files changed, 527 insertions(+), 112 deletions(-) create mode 100644 litellm/llms/model_inventory.py create mode 100644 tests/unit/llms/test_model_inventory.py diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 35107c595d2..8eec07dadda 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -169,18 +169,13 @@ ARRAY_KEYS: dict[str, JsonSchema] = { } INTEGER_KEYS: dict[str, JsonSchema] = { - "context_window": { - "type": "integer", - "minimum": 1, - "description": "Maximum combined input and generated output tokens, independent of the input and output limits.", - }, "max_tokens": { **NONNEG_INTEGER, "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.", }, "max_input_tokens": { **NONNEG_INTEGER, - "description": "Maximum input tokens, independent of the combined input and output context window.", + "description": "Maximum prompt/context tokens the model accepts.", }, "max_output_tokens": { **NONNEG_INTEGER, @@ -285,7 +280,7 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: "Gates whether a non-default temperature or the top_p/logprobs sampling params are " "accepted, which hold only when the effort resolves to 'none'." ), - "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], + "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], }, "comment": STRING, "audio_transcription_config": STRING, @@ -309,7 +304,7 @@ def classify(key: str, modes: tuple) -> Optional[JsonSchema]: def build_schema(prices: dict) -> JsonSchema: entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS} - all_keys = tuple(sorted({key for entry in entries.values() for key in entry} | {"context_window"})) + all_keys = tuple(sorted({key for entry in entries.values() for key in entry})) modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry})) unclassified = tuple(key for key in all_keys if classify(key, modes) is None) if unclassified: diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index 727fad68349..ec49186a88a 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -125,6 +125,22 @@ class _ChatGPTCatalog(BaseModel): error: object | None = None +def chatgpt_model_inventory_api_base(api_base: str | None = None, *, authenticator: Authenticator | None = None) -> str: + """Saved server OAuth credentials may only be sent to the server-configured backend.""" + auth: Final = authenticator if authenticator is not None else Authenticator() + trusted_base: Final = auth.get_api_base().rstrip("/") + if api_base is not None and api_base.rstrip("/") != trusted_base: + raise ValueError("Native model discovery requires the server-configured API base") + return trusted_base + + +def chatgpt_model_inventory_identity() -> str | None: + try: + return Authenticator().get_account_id() + except Exception: # noqa: BLE001 # unreadable server auth is not proof of an account change + return None + + async def get_chatgpt_model_info( *, model: str, @@ -153,8 +169,9 @@ async def get_chatgpt_model_inventory( ) -> SupplierModelInventory | SupplierInventoryUnavailable: auth: Final = authenticator if authenticator is not None else Authenticator() try: + trusted_base: Final = chatgpt_model_inventory_api_base(api_base, authenticator=auth) account: Final = await asyncio.to_thread(auth.get_account_id) - url: Final = f"{(api_base or auth.get_api_base()).rstrip('/')}/models?client_version={_CATALOG_CLIENT_VERSION}" + url: Final = f"{trusted_base}/models?client_version={_CATALOG_CLIENT_VERSION}" credential_scope: Final = hashlib.sha256(json.dumps((url, account)).encode()).hexdigest() if account else None except Exception: # noqa: BLE001 # malformed native auth is unavailable, never an authoritative empty inventory return SupplierInventoryUnavailable("authentication") diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py new file mode 100644 index 00000000000..5199b529c7e --- /dev/null +++ b/litellm/llms/model_inventory.py @@ -0,0 +1,110 @@ +"""Supplier-specific inventory dispatch shared by router and selected-offering discovery.""" + +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +import litellm +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.llms.chatgpt.model_info import ( + chatgpt_model_inventory_api_base, + chatgpt_model_inventory_identity, + get_chatgpt_model_inventory, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_DISCOVERY_PROVIDERS, get_openai_compatible_model_inventory +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.types.router import LiteLLM_Params + +_EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) +_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +@contextmanager +def prevent_supplier_device_login() -> Generator[None]: + with prevent_device_login(): + yield + + +def model_inventory_api_base(provider: str, api_base: str | None) -> str | None: + if provider == "chatgpt": + return chatgpt_model_inventory_api_base(api_base) + return api_base or { + "openrouter": "https://openrouter.ai/api/v1", + "vercel_ai_gateway": "https://ai-gateway.vercel.sh/v1", + }.get(provider) + + +def model_inventory_identity(provider: str) -> str | None: + return chatgpt_model_inventory_identity() if provider == "chatgpt" else None + + +async def get_provider_model_inventory( + *, + provider: str, + api_base: str | None, + headers: Mapping[str, str], + client: AsyncHTTPHandler, + cache: InMemoryCache, + force_refresh: bool = False, +) -> SupplierModelInventory | SupplierInventoryUnavailable: + if provider == "chatgpt": + return await get_chatgpt_model_inventory( + api_base=api_base, + client=client, + cache=cache, + force_refresh=force_refresh, + ) + resolved_base: Final = model_inventory_api_base(provider, api_base) + if resolved_base is None: + return SupplierInventoryUnavailable("malformed") + return await get_openai_compatible_model_inventory( + provider=provider, + api_base=resolved_base, + headers=headers, + client=client, + cache=cache, + force_refresh=force_refresh, + ) + + +async def get_deployment_model_metadata( + *, + params: LiteLLM_Params, + client: AsyncHTTPHandler, + cache: InMemoryCache, +) -> Mapping[str, object]: + if params.get("use_clientside_credentials") or "*" in params.model: + return _EMPTY_METADATA + if params.model.startswith("chatgpt/"): + model: str = params.model.removeprefix("chatgpt/") + provider: str = "chatgpt" + api_base: str | None = params.api_base + dynamic_api_key: str | None = None + else: + model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) + if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return _EMPTY_METADATA + api_key: Final = params.api_key or dynamic_api_key + headers: Final = _HEADERS.validate_python(params.get("extra_headers") or params.get("headers") or {}) + inventory: Final = await get_provider_model_inventory( + provider=provider, + api_base=api_base, + client=client, + cache=cache, + headers=MappingProxyType( + { + **({"authorization": f"Bearer {api_key}"} if api_key else {}), + **{key.lower(): value for key, value in headers.items()}, + } + ), + ) + return ( + inventory.models.get(model, _EMPTY_METADATA) + if isinstance(inventory, SupplierModelInventory) + else _EMPTY_METADATA + ) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index ae7240b8a7f..e4dbdd1bffa 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,4 +1,6 @@ import base64 +import hashlib +import hmac import os from collections.abc import Mapping from typing import Final, Literal, cast @@ -21,6 +23,7 @@ _V2_GCM_PREFIX: Final = "v2:gcm:" _ENCRYPTION_ALGORITHM_SETTING: Final = "encryption_algorithm" _ALGO_AES_GCM: Final = "aes-256-gcm" _ALGO_XSALSA20: Final = "xsalsa20-poly1305" +_MODEL_OFFERING_PROCESS_KEY: Final = os.urandom(32) def _get_salt_key(): @@ -34,6 +37,20 @@ def _get_salt_key(): return salt_key +def model_offering_identity_fingerprint(value: bytes) -> str: + """Opaque stable connection identity, never an unkeyed hash of supplier secrets. + + Use the same server-owned key source as encryption. With a configured salt or + master key identities survive process restarts; standalone/development callers + without either key use a private process key and do not retain that stability. + """ + configured_key: Final = _get_salt_key() + key: Final = ( + configured_key.encode() if isinstance(configured_key, str) and configured_key else _MODEL_OFFERING_PROCESS_KEY + ) + return hmac.new(key, b"litellm:model-offering-identity:v1\0" + value, hashlib.sha256).hexdigest() + + def _get_encryption_algorithm() -> str: """ Resolve the configured at-rest encryption algorithm for *new writes*. diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index 2fe89e7f859..55e8f28e624 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -14,12 +14,18 @@ import yaml from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.llms.chatgpt.authenticator import Authenticator, prevent_device_login -from litellm.llms.chatgpt.model_info import get_chatgpt_model_inventory from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY, get_openai_compatible_model_inventory +from litellm.llms.model_inventory import ( + get_provider_model_inventory, + model_inventory_api_base, + model_inventory_identity, + prevent_supplier_device_login, +) +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY +from litellm.proxy.common_utils.encrypt_decrypt_utils import model_offering_identity_fingerprint from litellm.proxy.offering_router import OfferingRouterView, OfferingServingSnapshot from litellm.router import Router +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.types.proxy.model_offerings import ModelOffering, ModelOfferingsConfig, SupplierConnection @@ -32,7 +38,7 @@ class _ResolvedConnection: api_key: str | None headers: Mapping[str, str] fingerprint: str - native_account_id: str | None = None + supplier_identity: str | None = None @dataclass(frozen=True, slots=True) @@ -63,21 +69,9 @@ def _resolve_secret(value: str) -> str: return resolved -def _native_account_id() -> str | None: - try: - return Authenticator().get_account_id() - except Exception: # noqa: BLE001 # an unreadable native auth file is not proof of an account change - return None - - def _resolve_connection(connection: SupplierConnection) -> _ResolvedConnection: - api_base: Final = ( - str(connection.api_base).rstrip("/") - if connection.api_base is not None - else { - "openrouter": "https://openrouter.ai/api/v1", - "vercel_ai_gateway": "https://ai-gateway.vercel.sh/v1", - }.get(connection.provider) + api_base: Final = model_inventory_api_base( + connection.provider, str(connection.api_base).rstrip("/") if connection.api_base is not None else None ) api_key: Final = _resolve_secret(connection.api_key.get_secret_value()) if connection.api_key is not None else None configured_headers: Final = { @@ -86,16 +80,16 @@ def _resolve_connection(connection: SupplierConnection) -> _ResolvedConnection: headers: Final = MappingProxyType( {**({"authorization": f"Bearer {api_key}"} if api_key is not None else {}), **configured_headers} ) - fingerprint: Final = hashlib.sha256( + fingerprint: Final = model_offering_identity_fingerprint( json.dumps((connection.provider, api_base, api_key, sorted(headers.items()))).encode() - ).hexdigest() + ) return _ResolvedConnection( connection.provider, api_base, api_key, headers, fingerprint, - _native_account_id() if connection.provider == "chatgpt" else None, + model_inventory_identity(connection.provider), ) @@ -113,7 +107,7 @@ def _load_config(path: Path) -> _LoadedConfig | _InvalidConfig: ( hashlib.sha256(content).hexdigest(), tuple( - (connection.fingerprint, connection.native_account_id) for connection in connections.values() + (connection.fingerprint, connection.supplier_identity) for connection in connections.values() ), ) ).encode() @@ -173,7 +167,9 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, deployment_id: Final = ( "offering-" + hashlib.sha256( - json.dumps((offering.model_name, connection.fingerprint, offering.upstream_model)).encode() + json.dumps( + (offering.model_name, connection.fingerprint, offering.upstream_model, connection.supplier_identity) + ).encode() ).hexdigest() ) return MappingProxyType( @@ -224,23 +220,22 @@ class ModelOfferingsManager: self.router = OfferingRouterView( OfferingServingSnapshot(template, frozenset(), MappingProxyType({}), frozenset()) ) + # Globally registered native affinity callbacks must read the same pinned + # snapshot as request routing, rather than the empty bootstrap template. + for callback in template.optional_callbacks or (): + if isinstance(callback, EncryptedContentAffinityCheck) and callback.router is template: + callback.router = self.router async def _fetch_provider(self, state: _ProviderState) -> _ProviderState: connection: Final = state.connection try: - fetched: Final = ( - await get_chatgpt_model_inventory( - client=self.client, cache=self.cache, api_base=connection.api_base, force_refresh=True - ) - if connection.provider == "chatgpt" - else await get_openai_compatible_model_inventory( - api_base=connection.api_base or "", - provider=connection.provider, - headers=connection.headers, - client=self.client, - cache=self.cache, - force_refresh=True, - ) + fetched: Final = await get_provider_model_inventory( + provider=connection.provider, + api_base=connection.api_base, + headers=connection.headers, + client=self.client, + cache=self.cache, + force_refresh=True, ) except Exception: # noqa: BLE001 # supplier failures cannot replace a successful inventory with absence return state @@ -267,7 +262,7 @@ class ModelOfferingsManager: if _unavailable_reason(offering, providers[offering.provider]) is None ) try: - with prevent_device_login(): + with prevent_supplier_device_login(): candidate: Final = ( self.template.snapshot_with_model_list(deployments, metadata_authoritative=True) if deployments != self.deployments @@ -331,8 +326,8 @@ class ModelOfferingsManager: if (previous := self.providers.get(name)) is not None and previous.connection.fingerprint == connection.fingerprint and ( - connection.native_account_id is None - or previous.connection.native_account_id == connection.native_account_id + connection.supplier_identity is None + or previous.connection.supplier_identity == connection.supplier_identity ) else _ProviderState(connection, None) for name, connection in loaded.connections.items() diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index c7e885eb6c2..b9f953b51d5 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -10,7 +10,7 @@ from starlette.types import ASGIApp, Receive, Scope, Send import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.llms.model_inventory import prevent_supplier_device_login from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator, alias_target, caller_alias_maps from litellm.router import Router @@ -36,6 +36,12 @@ _METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) class OfferingRouterView(Router): + """Request-pinned facade delegating native members to initialized Router instances. + + Router initialization belongs to the delegated instances. Reinitializing the + facade would allocate unused clients and register duplicate global callbacks. + """ + def __init__(self, snapshot: OfferingServingSnapshot) -> None: self._offering_snapshot = snapshot @@ -76,7 +82,7 @@ class OfferingRouterView(Router): def pin_snapshot(self) -> Generator[None]: token: Final = _PINNED_SNAPSHOT.set((id(self), self._offering_snapshot)) try: - with prevent_device_login(): + with prevent_supplier_device_login(): yield finally: _PINNED_SNAPSHOT.reset(token) @@ -98,6 +104,7 @@ class OfferingSnapshotMiddleware: class OfferingAccessGuard(CustomLogger): def __init__(self, router: OfferingRouterView, general_settings: Mapping[str, object] | None = None) -> None: + super().__init__() self.router = router self.general_settings = general_settings if general_settings is not None else {} @@ -139,6 +146,13 @@ class OfferingAccessGuard(CustomLogger): data: Mapping[str, object], call_type: CallTypesLiteral, ) -> Exception | None: + extra_body: Final = data.get("extra_body") + if isinstance(extra_body, Mapping) and "model" in extra_body: + return litellm.BadRequestError( + message="External offering mode does not accept supplier model overrides in extra_body", + model="", + llm_provider="", + ) if any( data.get(name) is not None for name in ( diff --git a/litellm/router.py b/litellm/router.py index 86784bdaf89..67583c1da65 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -127,10 +127,8 @@ from litellm.llms.base_llm.vector_store.transformation import ( from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.model_info import ( - MODEL_INFO_DISCOVERY_PROVIDERS, MODEL_INFO_REFRESH_CONCURRENCY, MODEL_INFO_REFRESH_SECONDS, - get_openai_compatible_model_info, ) from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy from litellm.router_strategy.budget_limiter import RouterBudgetLimiting @@ -10018,6 +10016,7 @@ class Router: snapshot: Final = copy.copy(self) snapshot.model_metadata_authoritative = metadata_authoritative snapshot.pattern_router = PatternMatchRouter() + snapshot.deployment_names = [] snapshot.provider_default_deployment_ids = [] snapshot._zero_cost_cache = {} snapshot._discovered_model_info_cache = InMemoryCache( @@ -10825,43 +10824,10 @@ class Router: async def _aget_deployment_model_metadata( self, params: LiteLLM_Params, *, client: AsyncHTTPHandler | None ) -> Mapping[str, object]: - if params.get("use_clientside_credentials") or "*" in params.model: - return MappingProxyType({}) - if params.model.startswith("chatgpt/"): - from litellm.llms.chatgpt.model_info import get_chatgpt_model_info + from litellm.llms.model_inventory import get_deployment_model_metadata - return await get_chatgpt_model_info( - model=params.model.removeprefix("chatgpt/"), - api_base=params.api_base, - client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), - cache=self.cache.in_memory_cache, - ) - model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) - if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: - return MappingProxyType({}) - discovery_api_base: Final = api_base or { - "openrouter": "https://openrouter.ai/api/v1", - "vercel_ai_gateway": "https://ai-gateway.vercel.sh/v1", - }.get(provider) - if discovery_api_base is None or "*" in model or params.get("use_clientside_credentials"): - return MappingProxyType({}) - api_key: Final = params.api_key or dynamic_api_key - headers: Final = TypeAdapter(Mapping[str, str]).validate_python( - params.get("extra_headers") or params.get("headers") or MappingProxyType({}) - ) - auth_headers: Final = ( - MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) - ) - return await get_openai_compatible_model_info( - model=model, - api_base=discovery_api_base, - provider=provider, - headers=MappingProxyType( - { - **auth_headers, - **MappingProxyType({key.lower(): value for key, value in headers.items()}), - } - ), + return await get_deployment_model_metadata( + params=params, client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), cache=self.cache.in_memory_cache, ) diff --git a/litellm/router_utils/model_request_defaults.py b/litellm/router_utils/model_request_defaults.py index b9b161270a0..f7870f8e5c1 100644 --- a/litellm/router_utils/model_request_defaults.py +++ b/litellm/router_utils/model_request_defaults.py @@ -18,17 +18,14 @@ class _Reasoning(BaseModel): class _RequestEffort(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) - reasoning_effort: str | int | None = None + reasoning_effort: str | int | _Reasoning | None = None reasoning: _Reasoning | None = None def effort(self) -> str | int | None: - return ( - self.reasoning_effort - if self.reasoning_effort is not None - else self.reasoning.effort - if self.reasoning is not None - else None + direct: Final = ( + self.reasoning_effort.effort if isinstance(self.reasoning_effort, _Reasoning) else self.reasoning_effort ) + return direct if direct is not None else self.reasoning.effort if self.reasoning is not None else None def model_request_defaults( diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 909a3f8850e..de3d760d57a 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -267,11 +267,6 @@ "comment": { "type": "string" }, - "context_window": { - "type": "integer", - "minimum": 1, - "description": "Maximum combined input and generated output tokens, independent of the input and output limits." - }, "cost_per_second": { "type": "number", "minimum": 0 @@ -285,8 +280,7 @@ "low", "medium", "high", - "xhigh", - "max" + "xhigh" ] }, "deprecation_date": { @@ -508,7 +502,7 @@ "max_input_tokens": { "type": "integer", "minimum": 0, - "description": "Maximum input tokens, independent of the combined input and output context window." + "description": "Maximum prompt/context tokens the model accepts." }, "max_output_tokens": { "type": "integer", diff --git a/tests/unit/llms/chatgpt/test_model_info.py b/tests/unit/llms/chatgpt/test_model_info.py index 98e2c818af3..e49f1f744a8 100644 --- a/tests/unit/llms/chatgpt/test_model_info.py +++ b/tests/unit/llms/chatgpt/test_model_info.py @@ -16,6 +16,7 @@ async def test_account_catalog_preserves_client_policy_separately_from_model_lim tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") auth_file: Final = tmp_path / "auth.json" auth_file.write_text( json.dumps({"access_token": "local-token", "account_id": "local-account", "expires_at": 10**30}) @@ -76,6 +77,7 @@ async def test_switching_accounts_invalidates_catalog_without_exposing_credentia tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") auth_file: Final = tmp_path / "auth.json" cache: Final = InMemoryCache() handler: Final = AsyncHTTPHandler() @@ -106,6 +108,7 @@ async def test_discovery_without_authorization_never_starts_device_login( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") with pytest.raises(GetAccessTokenError, match="model discovery cannot start device login"): Authenticator().get_access_token(allow_device_login=False) @@ -133,6 +136,7 @@ async def test_invalid_or_unavailable_catalog_preserves_fallback( status: int, body: dict[str, object], tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") (tmp_path / "auth.json").write_text(json.dumps({"access_token": "local", "expires_at": 10**30})) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -166,3 +170,28 @@ def test_codex_ui_efforts_are_not_advertised_as_api_efforts( ).metadata() assert metadata["reasoning_effort_levels"] == ("high", "max", "disabled") assert metadata["default_reasoning_effort"] == expected + + +async def test_caller_api_base_cannot_exfiltrate_saved_server_oauth(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import Mock + + from litellm.llms.chatgpt.model_info import get_chatgpt_model_inventory + from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable + + monkeypatch.setenv("CHATGPT_API_BASE", "https://trusted.test/backend-api/codex") + token: Final = Mock(side_effect=AssertionError("Rejected destinations must not even read the access token")) + monkeypatch.setattr(Authenticator, "get_access_token", token) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + + def reject(request: httpx.Request) -> httpx.Response: + raise AssertionError("Saved OAuth must not reach a caller-controlled destination") + + async with httpx.AsyncClient(transport=httpx.MockTransport(reject)) as client: + handler.client = client + result: Final = await get_chatgpt_model_inventory( + client=handler, cache=InMemoryCache(), api_base="https://attacker.test/capture" + ) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == "authentication" + token.assert_not_called() diff --git a/tests/unit/llms/test_model_inventory.py b/tests/unit/llms/test_model_inventory.py new file mode 100644 index 00000000000..8b0a7ed5ef6 --- /dev/null +++ b/tests/unit/llms/test_model_inventory.py @@ -0,0 +1,31 @@ +from typing import Final + +import pytest + +from litellm.llms.chatgpt.authenticator import Authenticator +from litellm.llms.model_inventory import model_inventory_api_base, model_inventory_identity + + +@pytest.mark.parametrize( + "provider,expected", + [ + ("openrouter", "https://openrouter.ai/api/v1"), + ("vercel_ai_gateway", "https://ai-gateway.vercel.sh/v1"), + ("openai", None), + ], +) +def test_inventory_connection_defaults_and_explicit_endpoints(provider, expected): + assert model_inventory_api_base(provider, None) == expected + assert model_inventory_api_base(provider, "https://configured.test/v1") == "https://configured.test/v1" + assert model_inventory_identity(provider) is None + + +def test_native_inventory_registry_uses_server_owned_backend_and_identity(monkeypatch: pytest.MonkeyPatch) -> None: + trusted: Final = "https://server-owned.test/backend-api/codex" + monkeypatch.setenv("CHATGPT_API_BASE", trusted) + monkeypatch.setattr(Authenticator, "get_account_id", lambda self: "fixture-account") + assert model_inventory_api_base("chatgpt", None) == trusted + assert model_inventory_api_base("chatgpt", trusted + "/") == trusted + assert model_inventory_identity("chatgpt") == "fixture-account" + with pytest.raises(ValueError, match="server-configured API base"): + model_inventory_api_base("chatgpt", "https://caller-owned.test/capture") diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 7c07f19d72f..c4596abe41d 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -266,3 +266,21 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +def test_model_offering_identity_uses_server_key_and_not_a_public_credential_hash(monkeypatch): + import hashlib + import hmac + + from litellm.proxy.common_utils.encrypt_decrypt_utils import model_offering_identity_fingerprint + + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") + low_entropy_header = b'Basic dXNlcjpwYXNz' + identity = model_offering_identity_fingerprint(low_entropy_header) + assert identity == hmac.new( + b"fixture-server-owned-key", b"litellm:model-offering-identity:v1\0" + low_entropy_header, hashlib.sha256 + ).hexdigest() + assert identity != hashlib.sha256(low_entropy_header).hexdigest() + assert identity != model_offering_identity_fingerprint(b'Basic dXNlcjpuZXctcGFzcw==') + monkeypatch.setenv("LITELLM_SALT_KEY", "other-server-owned-key") + assert identity != model_offering_identity_fingerprint(low_entropy_header) diff --git a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py index b848c4f1976..a850b3a0d83 100644 --- a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py @@ -1137,6 +1137,11 @@ async def test_v1_models_team_alias_inherits_token_limits_and_chat_mode(monkeypa "mode": "chat", "max_input_tokens": 876000, "max_output_tokens": 128000, + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supported_endpoints": ["/v1/chat/completions", "/v1/batch", "/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], "metadata": {"fallbacks": []}, } ] @@ -1183,6 +1188,7 @@ async def test_v1_models_team_image_alias_inherits_image_generation_mode(monkeyp "created": 1677610602, "owned_by": "openai", "mode": "image_generation", + "supported_endpoints": ["/v1/images/generations", "/v1/images/edits"], } ] diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 204052e4188..49df662b93e 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -3,8 +3,8 @@ from typing import Final import httpx import pytest -from openai import AsyncOpenAI import yaml +from openai import AsyncOpenAI from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy.model_offerings import ModelOfferingsManager @@ -196,9 +196,13 @@ async def test_snapshot_factory_dispatch_and_reload_resources_are_isolated() -> response: Final = await first.aresponses(model="selected", input="fixture", mock_response="local-only") assert response.output[0].content[0].text == "local-only" assert template.get_model_names() == [] + assert template.deployment_names == [] + assert first.deployment_names == ["openai/fixture"] references: Final = tuple(weakref.ref(template.snapshot_with_model_list(models)) for _ in range(30)) gc.collect() assert all(reference() is None for reference in references) + assert template.deployment_names == [] + assert first.deployment_names == ["openai/fixture"] assert callback_counts == tuple( len(callbacks) for callbacks in ( @@ -335,3 +339,139 @@ async def test_native_client_forwards_one_supplier_authorization_value( model="selected", messages=[{"role": "user", "content": "authenticate"}], client=sdk_client ) assert response.choices[0].message.content == "accepted" + + +async def test_global_affinity_preserves_encrypted_content_across_switch_and_pinned_reload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm.responses.utils import ResponsesAPIRequestUtils + + monkeypatch.setattr(litellm, "callbacks", []) + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "providers": { + "source": {"provider": "openai", "api_base": "https://supplier.test/v1", "api_key": "fixture-key"} + }, + "offerings": [ + {"model_name": name, "source": "manual", "provider": "source", "upstream_model": "gpt-4o-mini"} + for name in ("parent", "child") + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + template: Final = Router(model_list=[], optional_pre_call_checks=["encrypted_content_affinity"]) + manager: Final = ModelOfferingsManager(path=path, template=template, client=handler) + assert await manager.reload(initial=True) + old: Final = manager.router.latest_snapshot() + origin: Final = old.router.get_model_ids("parent")[0] + wrapped: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("opaque-fixture", origin) + with manager.router.pin_snapshot(): + config["offerings"] = config["offerings"][1:] + path.write_text(yaml.safe_dump(config)) + assert await manager.reload() + request: Final = { + "input": [{"type": "reasoning", "encrypted_content": wrapped}, {"role": "user", "content": "fixture"}] + } + child: Final = [row for row in old.router.get_model_list() or [] if row["model_name"] == "child"] + result: Final = await manager.router.async_callback_filter_deployments( + model="child", + healthy_deployments=child, + messages=None, + parent_otel_span=None, + request_kwargs=request, + ) + assert result == child + assert request["input"][0]["encrypted_content"] == wrapped + next_request: Final = { + "input": [{"type": "reasoning", "encrypted_content": wrapped}, {"role": "user", "content": "fixture"}] + } + await manager.router.async_callback_filter_deployments( + model="child", + healthy_deployments=child, + messages=None, + parent_otel_span=None, + request_kwargs=next_request, + ) + assert next_request["input"] == [{"role": "user", "content": "fixture"}] + assert template.deployment_names == [] + + +async def test_private_connection_identity_is_stable_and_changes_on_credentials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "providers": { + "source": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "headers": {"Authorization": "Basic dXNlcjpwYXNz"}, + } + }, + "offerings": [ + {"model_name": "selected", "source": "manual", "provider": "source", "upstream_model": "gpt-4o-mini"} + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) + assert await manager.reload(initial=True) + before: Final = manager.router.get_model_ids() + assert await manager.reload(force_inventory=True) + assert manager.router.get_model_ids() == before + config["providers"]["source"]["headers"]["Authorization"] = "Basic dXNlcjpuZXctcGFzcw==" + path.write_text(yaml.safe_dump(config)) + assert await manager.reload() + assert manager.router.get_model_ids() != before + + +async def test_native_account_change_changes_private_deployment_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.chatgpt.authenticator import Authenticator + + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") + monkeypatch.setenv("CHATGPT_API_BASE", "https://native.test/backend-api/codex") + account: Final = {"id": "first"} + monkeypatch.setattr(Authenticator, "get_account_id", lambda self: account["id"]) + monkeypatch.setattr(Authenticator, "get_access_token", lambda self, **kwargs: "fixture-oauth") + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": {"native": {"provider": "chatgpt"}}, + "offerings": [ + {"model_name": "selected", "source": "auto", "provider": "native", "upstream_model": "fixture"} + ], + } + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"models": [{"slug": "fixture"}]})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) + assert await manager.reload(initial=True) + first: Final = manager.router.get_model_ids() + assert len(first) == 1 + account["id"] = "second" + assert await manager.reload(force_inventory=True) + assert manager.router.get_model_ids() != first + assert manager.router.latest_snapshot().available_models == {"selected"} diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 6b21891f7c7..575da05b2e5 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -1,8 +1,9 @@ from types import MappingProxyType from typing import Final -import litellm import pytest + +import litellm from litellm.caching.caching import DualCache from litellm.proxy._types import ( LiteLLM_EndUserTable, @@ -146,3 +147,26 @@ async def test_public_and_router_aliases_follow_offering_availability_without_cr assert await guard.async_filter_listed_models(admin, ("fast",)) == ("fast",) legacy: Final = OfferingAccessGuard(view, {"use_team_public_model_name": False}) assert await legacy.async_filter_listed_models(auth, ("fast", active)) == (active,) + + +async def test_body_model_override_cannot_bypass_offering_authority() -> None: + view: Final = OfferingRouterView( + OfferingServingSnapshot(Router(model_list=[]), frozenset({"selected"}), MappingProxyType({}), frozenset()) + ) + guard: Final = OfferingAccessGuard(view) + assert guard.message_logging is True + assert guard.turn_off_message_logging is False + auth: Final = UserAPIKeyAuth() + for call_type in ("acompletion", "aresponses"): + assert isinstance( + await guard.async_pre_call_hook( + auth, DualCache(), {"model": "selected", "extra_body": {"model": "unselected"}}, call_type + ), + litellm.BadRequestError, + ) + assert ( + await guard.async_pre_call_hook( + auth, DualCache(), {"model": "selected", "extra_body": {"metadata": {"fixture": "allowed"}}}, call_type + ) + is None + ) diff --git a/tests/unit/router_utils/test_model_request_defaults.py b/tests/unit/router_utils/test_model_request_defaults.py index 71762c8dd8c..1b0a05448b4 100644 --- a/tests/unit/router_utils/test_model_request_defaults.py +++ b/tests/unit/router_utils/test_model_request_defaults.py @@ -94,3 +94,14 @@ def test_request_budget_maps_are_immutable_and_only_common_policy_is_advertised( assert defaults is not None with pytest.raises(TypeError): defaults.output_token_budget_by_reasoning_effort["max"] = 1 + + +@pytest.mark.parametrize("surface,key", [("acompletion", "max_tokens"), ("aresponses", "max_output_tokens")]) +def test_structured_reasoning_effort_preserves_summary_and_caller_precedence(surface, key): + metadata: Final = {"request_defaults": _POLICY} + deployment: Final = {"reasoning_effort": {"effort": "max", "summary": "auto"}} + request: Final = {"reasoning_effort": {"effort": "high", "summary": "auto"}} + assert model_request_defaults(metadata, deployment, request, surface) == {key: 65536} + assert model_request_defaults(metadata, deployment, {}, surface) == {key: 131072} + assert request == {"reasoning_effort": {"effort": "high", "summary": "auto"}} + assert deployment == {"reasoning_effort": {"effort": "max", "summary": "auto"}} diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 295d2e51023..c6a43787079 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -51,6 +51,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/litellm_core_utils/specialty_caches/service_trace_id_cache.py", "litellm/llms/azure/common_utils.py", "litellm/llms/bedrock/base_aws_llm.py", + "litellm/llms/chatgpt/model_info.py", "litellm/llms/custom_httpx/http_handler.py", "litellm/llms/gigachat/authenticator.py", "litellm/llms/litellm_proxy/skills/handler.py", diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 8655f0c5323..6eee39b7628 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -585,13 +585,6 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): ] -@pytest.mark.parametrize("context", [0, -1, True, 1.5, "1200"]) -def test_schema_rejects_invalid_total_context(committed_schema, context): - assert not build_validator(committed_schema).is_valid( - {"fixture-model": {"litellm_provider": "openai", "context_window": context}} - ) - - def test_schema_and_model_info_preserve_independent_total_context(committed_schema, monkeypatch): entry = { "litellm_provider": "openai", @@ -600,6 +593,8 @@ def test_schema_and_model_info_preserve_independent_total_context(committed_sche "max_output_tokens": 200, "max_tokens": 200, } + # The trusted-base generated catalog schema permits optional unknown properties. + # Runtime metadata supplies and validates the separate total-context contract. assert build_validator(committed_schema).is_valid({"fixture-model": entry}) monkeypatch.setattr(litellm, "model_cost", {"fixture-model": entry}) info = litellm.get_model_info("fixture-model") diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 6ef00db5764..66365e0c3a3 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -3345,3 +3345,58 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info( assert router.cached_model_group_info("grp").input_cost_per_token == new_price assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price + + +@pytest.mark.parametrize( + "params", + ( + LiteLLM_Params(model="openai/*", api_base="https://fixture.test/v1"), + LiteLLM_Params(model="openai/fixture", api_base="https://fixture.test/v1", use_clientside_credentials=True), + LiteLLM_Params(model="anthropic/claude-sonnet-4-5"), + ), +) +async def test_deployment_metadata_skips_unresolved_or_unsupported_supplier_connections(params: LiteLLM_Params) -> None: + router: Final = Router(model_list=[]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + + def unexpected(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"No catalog request expected for {request.url.path}") + + async with httpx.AsyncClient(transport=httpx.MockTransport(unexpected)) as http_client: + handler.client = http_client + assert await router._aget_deployment_model_metadata(params, client=handler) == {} + + +async def test_deployment_metadata_uses_resolved_catalog_credentials_and_independent_limits() -> None: + router: Final = Router(model_list=[]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + paths: Final[list[str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + paths.append(request.url.path) + assert request.headers["authorization"] == "Bearer operator-override" + assert request.headers["x-provider-route"] == "fixture" + return httpx.Response( + 200, + json={ + "data": [{"id": "fixture", "context_window": 1200, "max_input_tokens": 1100, "max_output_tokens": 200}] + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler.client = http_client + metadata: Final = await router._aget_deployment_model_metadata( + LiteLLM_Params( + model="openai/fixture", + api_base="https://fixture.test/v1", + api_key="default-key", + extra_headers={"Authorization": "Bearer operator-override", "x-provider-route": "fixture"}, + ), + client=handler, + ) + assert paths == ["/v1/models"] + assert metadata["context_window"] == 1200 + assert metadata["max_input_tokens"] == 1100 + assert metadata["max_output_tokens"] == 200 From b2d662083e79e788cfec75db51e0a726ce573a11 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 11:52:08 +0200 Subject: [PATCH 09/34] docs(proxy): explain offering security and identity upgrades --- examples/model_offerings/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index de4c756be2f..d2734f2f9e7 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -60,3 +60,12 @@ Startup validates the complete file and awaits initial inventory requests. AUTO The file is checked every five seconds by default, with a configurable range of 1 to 300 seconds. Inventories refresh every 300 seconds by default, with a range of 1 to 86400 seconds. Changing that interval triggers a fresh inventory request and resets its deadline. Invalid file edits retain the whole last-valid serving configuration, including credentials, and log only the error kind. Unselected inventory entries never become public routes. Caller connection overrides and runtime model-management mutations are rejected in external mode; edit the external file instead Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The inventory HTTP client and polling task are closed at proxy shutdown after requests drain + + +Native subscription inventory uses the server-owned backend from `CHATGPT_API_BASE` / `OPENAI_CHATGPT_API_BASE`, or the provider default. An explicit connection URL must match that backend; per-model destinations cannot receive saved server OAuth credentials. Discovery never starts device login. Supplier-specific inventory behavior lives in the shared `litellm/llms/model_inventory.py` dispatcher. + +Connection identities use HMAC with the existing private server salt/master key. With that key configured, deployment IDs remain stable across restarts and change when supplier credentials or native account identity changes. No additional configured secret is required. Unconfigured standalone/development mode uses a private process key and does not promise ID stability across restart. Upgrading from earlier unkeyed offering IDs changes IDs once: encrypted reasoning tied to a removed ID may be discarded on replay or model switch, while readable history and model limits/defaults remain available. Native affinity callbacks use the request-pinned serving view during reloads. + +Caller `extra_body.model` overrides are rejected before supplier forwarding; harmless extra body fields remain supported. Structured `reasoning_effort` objects such as `{effort: high, summary: auto}` select the same configured output budget as the corresponding effort string. + +The optional bundled cost-map JSON-schema extension is deferred: its trusted-base guard validates PR artifacts with the base generator. The runtime metadata/configuration contract validates separate total/input/output limits and defaults independently, and unknown optional bundled catalog properties remain forward compatible. From 098deab58b4af9eb5888e197497cd8e9c4c6af4a Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 12:08:52 +0200 Subject: [PATCH 10/34] fix(proxy): reuse cached model inventory HTTP client --- litellm/proxy/proxy_server.py | 11 ++-- litellm/types/llms/custom_http.py | 1 + .../unit/proxy/proxy_server/test_lifecycle.py | 62 +++++++++++++++++++ 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c890646975d..3d10b857951 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -334,7 +334,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, get_async_httpx_client from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._experimental.mcp_server.byok_credential_cache import byok_credential_cache @@ -725,6 +725,7 @@ from litellm.proxy.spend_tracking.spend_event_producer import ( SpendEventProducer, build_spend_event_producer, ) +from litellm.types.llms.custom_http import httpxSpecialProvider try: from litellm.proxy.enterprise_billing.billing_metrics import ( @@ -1598,7 +1599,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState cast(object, general_settings.get("model_offerings_path")), # cast-ok: validate untyped config strict=True, ) - offerings_client: Final = AsyncHTTPHandler() if offerings_path is not None else None + offerings_client: Final = ( + get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelInventory) if offerings_path is not None else None + ) offerings_manager: Final = ( ModelOfferingsManager( path=pathlib.Path(str(offerings_path)), @@ -1610,10 +1613,8 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState ) if offerings_manager is not None and offerings_client is not None: if llm_model_list or (llm_router is not None and llm_router.get_model_ids()) or store_model_in_db: - await offerings_client.close() raise ValueError("External offering mode requires an empty model_list and store_model_in_db disabled") if not await offerings_manager.reload(initial=True): - await offerings_client.close() raise ValueError("Initial external offering configuration is invalid") llm_router = offerings_manager.router offerings_guard: Final = ( @@ -1683,8 +1684,6 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState await asyncio.gather(offerings_task, return_exceptions=True) if offerings_guard is not None: litellm.logging_callback_manager.remove_callback_from_all_lists(offerings_guard, require_self=True) - if offerings_client is not None: - await offerings_client.close() # Shutdown event - close shared aiohttp session if shared_aiohttp_session is not None: diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 47f80c52845..65f72be2d45 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -34,6 +34,7 @@ class httpxSpecialProvider(str, Enum): ROICalculator = "roi_calculator" Sandbox = "sandbox" ModelCostMap = "model_cost_map" + ModelInventory = "model_inventory" PasswordBreachCheck = "password_breach_check" ASGI = "asgi" AgentHarness = "agent_harness" diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index ba5501315d9..f522d7ab0f9 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -1260,3 +1260,65 @@ async def test_prometheus_fallback_stats_job_runs_when_the_lock_is_free_or_absen await jobs["prometheus_fallback_stats_job"]() assert send_fallback_stats.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("configuration", ["valid", "invalid", "mixed"]) +async def test_external_inventory_lifespan_does_not_close_shared_clients(tmp_path, monkeypatch, configuration): + import httpx + import yaml + + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.router import Router + from litellm.types.llms.custom_http import httpxSpecialProvider + + path = tmp_path / "offerings.yaml" + path.write_text( + "invalid: true\n" + if configuration == "invalid" + else yaml.safe_dump( + { + "version": 1, + "providers": {"source": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "manual", "source": "manual", "provider": "source", "upstream_model": "backend"} + ], + } + ) + ) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + inventory = get_async_httpx_client(httpxSpecialProvider.ModelInventory) + inference = get_async_httpx_client("openai") + assert inventory is not inference + await inventory.client.aclose() + inventory.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200))) + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + monkeypatch.setattr(ps, "llm_router", Router(model_list=[])) + monkeypatch.setattr(ps, "llm_model_list", [] if configuration != "mixed" else [{"model_name": "unmanaged"}]) + monkeypatch.setattr(ps, "store_model_in_db", False) + monkeypatch.setattr(ps, "scheduler", None) + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + ps.general_settings, {"model_offerings_path": str(path), "disable_model_info_refresh": True}, clear=True + ), + ): + try: + if configuration == "valid": + async with proxy_startup_event(app=None): + assert ps.llm_router.model_names == {"manual"} + else: + with pytest.raises(ValueError, match="external offering|External offering"): + async with proxy_startup_event(app=None): + pytest.fail("Invalid external config must reject startup") + assert get_async_httpx_client(httpxSpecialProvider.ModelInventory) is inventory + assert not inventory.client.is_closed + assert not inference.client.is_closed + assert (await inventory.get("https://supplier.test/v1/models")).status_code == 200 + finally: + await inventory.close() + await inference.close() From f7178be564702ad4d248b66db784111e24a2ed57 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 12:12:18 +0200 Subject: [PATCH 11/34] fix(proxy): refresh cached inventory client leases --- examples/model_offerings/README.md | 2 +- litellm/proxy/model_offerings.py | 7 +-- litellm/proxy/proxy_server.py | 11 ++--- .../unit/proxy/proxy_server/test_lifecycle.py | 2 +- tests/unit/proxy/test_model_offerings.py | 46 +++++++++++++++++++ 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index d2734f2f9e7..24117b4c5b0 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -59,7 +59,7 @@ Startup validates the complete file and awaits initial inventory requests. AUTO The file is checked every five seconds by default, with a configurable range of 1 to 300 seconds. Inventories refresh every 300 seconds by default, with a range of 1 to 86400 seconds. Changing that interval triggers a fresh inventory request and resets its deadline. Invalid file edits retain the whole last-valid serving configuration, including credentials, and log only the error kind. Unselected inventory entries never become public routes. Caller connection overrides and runtime model-management mutations are rejected in external mode; edit the external file instead -Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The inventory HTTP client and polling task are closed at proxy shutdown after requests drain +Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The polling task stops at proxy shutdown after requests drain. Inventory refreshes reacquire the dedicated shared HTTP client through the native cache, whose eviction lifecycle owns client closure Native subscription inventory uses the server-owned backend from `CHATGPT_API_BASE` / `OPENAI_CHATGPT_API_BASE`, or the provider default. An explicit connection URL must match that backend; per-model destinations cannot receive saved server OAuth credentials. Discovery never starts device login. Supplier-specific inventory behavior lives in the shared `litellm/llms/model_inventory.py` dispatcher. diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index 55e8f28e624..9794539e527 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -14,7 +14,7 @@ import yaml from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.model_inventory import ( get_provider_model_inventory, model_inventory_api_base, @@ -26,6 +26,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import model_offering_iden from litellm.proxy.offering_router import OfferingRouterView, OfferingServingSnapshot from litellm.router import Router from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.types.proxy.model_offerings import ModelOffering, ModelOfferingsConfig, SupplierConnection @@ -201,7 +202,7 @@ class ModelOfferingsManager: *, path: Path, template: Router, - client: AsyncHTTPHandler, + client: AsyncHTTPHandler | None = None, clock: Callable[[], float] = time.monotonic, ) -> None: self.path = path @@ -233,7 +234,7 @@ class ModelOfferingsManager: provider=connection.provider, api_base=connection.api_base, headers=connection.headers, - client=self.client, + client=self.client or get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelInventory), cache=self.cache, force_refresh=True, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3d10b857951..0abc0edceb1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -334,7 +334,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, get_async_httpx_client +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._experimental.mcp_server.byok_credential_cache import byok_credential_cache @@ -725,7 +725,6 @@ from litellm.proxy.spend_tracking.spend_event_producer import ( SpendEventProducer, build_spend_event_producer, ) -from litellm.types.llms.custom_http import httpxSpecialProvider try: from litellm.proxy.enterprise_billing.billing_metrics import ( @@ -1599,19 +1598,15 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState cast(object, general_settings.get("model_offerings_path")), # cast-ok: validate untyped config strict=True, ) - offerings_client: Final = ( - get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelInventory) if offerings_path is not None else None - ) offerings_manager: Final = ( ModelOfferingsManager( path=pathlib.Path(str(offerings_path)), template=llm_router or Router(model_list=[]), - client=offerings_client, ) - if offerings_client is not None + if offerings_path is not None else None ) - if offerings_manager is not None and offerings_client is not None: + if offerings_manager is not None: if llm_model_list or (llm_router is not None and llm_router.get_model_ids()) or store_model_in_db: raise ValueError("External offering mode requires an empty model_list and store_model_in_db disabled") if not await offerings_manager.reload(initial=True): diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index f522d7ab0f9..c086df4f14c 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -1312,7 +1312,7 @@ async def test_external_inventory_lifespan_does_not_close_shared_clients(tmp_pat async with proxy_startup_event(app=None): assert ps.llm_router.model_names == {"manual"} else: - with pytest.raises(ValueError, match="external offering|External offering"): + with pytest.raises(ValueError, match=r"external offering|External offering"): async with proxy_startup_event(app=None): pytest.fail("Invalid external config must reject startup") assert get_async_httpx_client(httpxSpecialProvider.ModelInventory) is inventory diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 49df662b93e..6b90b459b95 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -475,3 +475,49 @@ async def test_native_account_change_changes_private_deployment_identity( assert await manager.reload(force_inventory=True) assert manager.router.get_model_ids() != first assert manager.router.latest_snapshot().available_models == {"selected"} + + +async def test_shared_inventory_client_is_reacquired_after_cache_eviction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": {"source": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "selected", "source": "auto", "provider": "source", "upstream_model": "backend"} + ], + } + ) + ) + cache: Final = LLMClientCache() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", cache) + first: Final = get_async_httpx_client(httpxSpecialProvider.ModelInventory) + await first.client.aclose() + first.client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [{"id": "backend"}]})) + ) + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), clock=lambda: 0.0) + assert await manager.reload(initial=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + cache.delete_cache(cache.update_cache_key_with_event_loop("async_httpx_clientmodel_inventory")) + await first.close() + second: Final = get_async_httpx_client(httpxSpecialProvider.ModelInventory) + assert second is not first + await second.client.aclose() + second.client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) + try: + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == set() + assert not second.client.is_closed + finally: + await second.close() From d8418971fe6864d3ffca68f5c4314001a23efdc4 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Wed, 7 Oct 2026 14:15:00 +0200 Subject: [PATCH 12/34] fix(proxy): keep supplier headers private in model discovery --- .../common_utils/openai_endpoint_utils.py | 34 +++--- .../test_openai_endpoint_utils.py | 30 ++++-- .../proxy_server/test_routes_model_info.py | 100 ++++++++++++++++++ 3 files changed, 142 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/common_utils/openai_endpoint_utils.py b/litellm/proxy/common_utils/openai_endpoint_utils.py index f85bbf3b380..c2272c75fe1 100644 --- a/litellm/proxy/common_utils/openai_endpoint_utils.py +++ b/litellm/proxy/common_utils/openai_endpoint_utils.py @@ -17,21 +17,27 @@ def remove_sensitive_info_from_deployment( excluded_keys: set[str] | None = None, ) -> dict: """ - Removes sensitive information from a deployment dictionary. + Returns a public deployment copy without credentials or supplier headers. Args: deployment_dict (dict): The deployment dictionary to remove sensitive information from. excluded_keys (Optional[Set[str]]): Set of keys that should not be masked (exact match). Returns: - dict: The modified deployment dictionary with sensitive information removed. + dict: A copied deployment dictionary with sensitive information removed. """ - deployment_dict["litellm_params"].pop("api_key", None) - deployment_dict["litellm_params"].pop("client_secret", None) - deployment_dict["litellm_params"].pop("vertex_credentials", None) - deployment_dict["litellm_params"].pop("vertex_ai_credentials", None) - deployment_dict["litellm_params"].pop("aws_access_key_id", None) - deployment_dict["litellm_params"].pop("aws_secret_access_key", None) + _private_params: Final = frozenset( + ( + "api_key", + "client_secret", + "vertex_credentials", + "vertex_ai_credentials", + "aws_access_key_id", + "aws_secret_access_key", + "extra_headers", + "headers", + ) + ) # Rate-limit config fields must never be masked — they are integers, not credentials. # The field names contain "key" which matches the masker's sensitive pattern, so we @@ -42,11 +48,13 @@ def remove_sensitive_info_from_deployment( } _excluded: Final = (excluded_keys or set()) | _rate_limit_config_keys - deployment_dict["litellm_params"] = SENSITIVE_DATA_MASKER.mask_dict( - deployment_dict["litellm_params"], excluded_keys=_excluded - ) - - return deployment_dict + return { + **deployment_dict, + "litellm_params": SENSITIVE_DATA_MASKER.mask_dict( + {key: value for key, value in deployment_dict["litellm_params"].items() if key not in _private_params}, + excluded_keys=_excluded, + ), + } async def get_custom_llm_provider_from_request_body(request: Request) -> str | None: diff --git a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py index 3e3e89f8117..1bc428649a0 100644 --- a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py +++ b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py @@ -77,9 +77,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ), ], ) -def test_remove_sensitive_info_from_deployment( - model_config: dict, expected_config: dict -): +def test_remove_sensitive_info_from_deployment(model_config: dict, expected_config: dict): sanitized_config = remove_sensitive_info_from_deployment(model_config) assert sanitized_config == expected_config @@ -105,15 +103,10 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): assert "*" in sanitized_config["litellm_params"]["access_token"] # With excluded_keys, litellm_credentials_name should NOT be masked. - # ``remove_sensitive_info_from_deployment`` mutates its input, so feed it - # a fresh copy rather than the already-sanitized one. sanitized_config = remove_sensitive_info_from_deployment( copy.deepcopy(base_config), excluded_keys={"litellm_credentials_name"} ) - assert ( - sanitized_config["litellm_params"]["litellm_credentials_name"] - == "my-credential-name" - ) + assert sanitized_config["litellm_params"]["litellm_credentials_name"] == "my-credential-name" # access_token should still be masked (not in excluded_keys) assert sanitized_config["litellm_params"]["access_token"] != "token-12345" @@ -121,3 +114,22 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): # api_key should still be removed (popped) regardless of excluded_keys assert "api_key" not in sanitized_config["litellm_params"] + + +def test_public_deployment_omits_all_headers_without_mutating_runtime_parameters(): + configured = { + "model_name": "selected", + "litellm_params": { + "model": "openai/backend", + "api_key": "private-key", + "extra_headers": {"X-Session": "opaque-session", "Trace-Value": "private-trace"}, + "headers": {"Unclassified": "another-secret"}, + "temperature": 0.25, + }, + } + original = copy.deepcopy(configured) + public = remove_sensitive_info_from_deployment(configured, excluded_keys={"extra_headers", "headers"}) + assert public["litellm_params"] == {"model": "openai/backend", "temperature": 0.25} + assert configured == original + assert public is not configured + assert public["litellm_params"] is not configured["litellm_params"] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 5f5a3099f62..cfe219cfb44 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -12,6 +12,7 @@ from __future__ import annotations import copy from collections.abc import Callable from contextlib import AbstractContextManager +from pathlib import Path from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -933,3 +934,102 @@ def test_v2_model_info_access_group_paginates_over_the_filtered_set(client, auth assert _model_names(payload) == ["openai/*"] assert payload["total_count"] == 2 assert payload["total_pages"] == 2 + + +async def test_external_headers_are_private_on_permitted_nonadmin_discovery( + tmp_path: Path, + mock_prisma: MagicMock, + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + import yaml + from openai import AsyncOpenAI + + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.model_offerings import ModelOfferingsManager + + secret: Final = "opaque-session-fixture" + monkeypatch.setenv("SUPPLIER_SESSION", secret) + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "source": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "api_key": "supplier-fixture-key", + "headers": {"X-Session": "os.environ/SUPPLIER_SESSION"}, + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "source", + "upstream_model": "fixture-backend", + "model_info": {"mode": "chat", "context_window": 1000}, + } + ], + } + ) + ) + catalog_handler: Final = AsyncHTTPHandler() + await catalog_handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as catalog_client: + catalog_handler.client = catalog_client + manager: Final = ModelOfferingsManager( + path=path, template=litellm.Router(model_list=[], num_retries=0), client=catalog_handler + ) + assert await manager.reload(initial=True) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(proxy_server, "user_config_file_path", None) + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + monkeypatch.setattr(proxy_server, "llm_router", manager.router) + monkeypatch.setattr(proxy_server, "llm_model_list", manager.router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + deployment_id: Final = manager.router.get_model_ids()[0] + with auth_as(LitellmUserRoles.INTERNAL_USER, models=["selected"]): + for route, query in ( + ("/model/info", {}), + ("/v1/model/info", {"litellm_model_id": deployment_id}), + ("/v2/model/info", {}), + ("/v2/model/info", {"modelId": deployment_id}), + ("/model_group/info", {}), + ("/v1/models", {}), + ("/v1/models/selected", {}), + ): + response: Final = client.get(route, params=query) + assert response.status_code == 200, response.text + assert "selected" in response.text + assert secret not in response.text + assert "x-session" not in response.text.lower() + runtime_headers: Final = manager.router.get_model_list()[0]["litellm_params"]["extra_headers"] + assert runtime_headers["x-session"] == secret + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["x-session"] == secret + return httpx.Response( + 200, + json={ + "id": "fixture-response", + "object": "chat.completion", + "created": 1, + "model": "fixture-backend", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "accepted"}, "finish_reason": "stop"} + ], + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + async with AsyncOpenAI(api_key="supplier-fixture-key", http_client=http_client, max_retries=0) as sdk_client: + result: Final = await manager.router.acompletion( + model="selected", messages=[{"role": "user", "content": "fixture"}], client=sdk_client + ) + assert result.choices[0].message.content == "accepted" From 6fc08a4644b62eb25a607585859cbe340795953f Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 8 Oct 2026 11:07:15 +0200 Subject: [PATCH 13/34] refactor(proxy): satisfy upstream discovery type discipline --- litellm/llms/chatgpt/model_info.py | 4 +++- litellm/llms/model_inventory.py | 17 ++++++++--------- litellm/llms/openai_like/model_info.py | 10 ++++++++++ litellm/proxy/_types.py | 4 +++- litellm/proxy/offering_router.py | 6 +++++- litellm/proxy/proxy_server.py | 4 ++-- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index ec49186a88a..99e4a63ca7e 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -22,6 +22,8 @@ _EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) class _ReasoningLevel(BaseModel): + model_config = ConfigDict(frozen=True) + effort: str @@ -210,10 +212,10 @@ async def get_chatgpt_model_inventory( MappingProxyType({card.slug: card.metadata() for card in catalog.models}), credential_scope ) cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) - return inventory except httpx.HTTPStatusError: return SupplierInventoryUnavailable("http", credential_scope) except ValidationError: return SupplierInventoryUnavailable("malformed", credential_scope) except Exception: # noqa: BLE001 # optional metadata discovery must not interrupt proxy startup return SupplierInventoryUnavailable("transport", credential_scope) + return inventory diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py index 5199b529c7e..2f065f823fe 100644 --- a/litellm/llms/model_inventory.py +++ b/litellm/llms/model_inventory.py @@ -80,15 +80,14 @@ async def get_deployment_model_metadata( ) -> Mapping[str, object]: if params.get("use_clientside_credentials") or "*" in params.model: return _EMPTY_METADATA - if params.model.startswith("chatgpt/"): - model: str = params.model.removeprefix("chatgpt/") - provider: str = "chatgpt" - api_base: str | None = params.api_base - dynamic_api_key: str | None = None - else: - model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) - if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: - return _EMPTY_METADATA + connection: Final = ( + (params.model.removeprefix("chatgpt/"), "chatgpt", None, params.api_base) + if params.model.startswith("chatgpt/") + else litellm.get_llm_provider(model=params.model, litellm_params=params) + ) + model, provider, dynamic_api_key, api_base = connection + if provider != "chatgpt" and provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return _EMPTY_METADATA api_key: Final = params.api_key or dynamic_api_key headers: Final = _HEADERS.validate_python(params.get("extra_headers") or params.get("headers") or {}) inventory: Final = await get_provider_model_inventory( diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 9d6683a2534..012d00c290f 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -34,26 +34,36 @@ _TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] class _Modalities(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + input: Sequence[str] | None = None output: Sequence[str] | None = None class _Architecture(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + input_modalities: Sequence[str] | None = None output_modalities: Sequence[str] | None = None class _TopProvider(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + context_length: _TokenLimit = None max_completion_tokens: _TokenLimit = None class _Reasoning(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + supported_efforts: Sequence[str] | None = None default_effort: str | None = None class _ReasoningOption(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + type: str values: Sequence[str] | None = None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 33fd535798f..58f48ee85f1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1225,7 +1225,9 @@ class ModelInfoDelete(LiteLLMPydanticObjectBase): id: str -class ModelInfo(GatewayModelMetadata, LiteLLMPydanticObjectBase): +class ModelInfo( # frozen-ok: native model management assigns team_public_model_name after validation + GatewayModelMetadata, LiteLLMPydanticObjectBase +): id: str | None mode: Literal["embedding", "chat", "completion"] | None input_cost_per_token: float | None = 0.0 diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index b9f953b51d5..7998f8c8de1 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -63,7 +63,11 @@ class OfferingRouterView(Router): return setattr(self.serving_snapshot().router, name, value) - def _offering_reject_mutation(self, *args: object, **kwargs: object) -> None: + def _offering_reject_mutation( + self, + *args: object, + **kwargs: object, # kwargs-ok: blocks all native Router mutation signatures + ) -> None: raise litellm.BadRequestError( message="Model definitions are owned by the external offering configuration", model="", llm_provider="" ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dd94704b7dd..2e744fe15b6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1344,11 +1344,11 @@ async def _call_current_lens_signal_router( @asynccontextmanager async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]: + global llm_router # noqa: PLW0603 # proxy routes share the external router published during lifespan global \ prisma_client, \ master_key, \ use_background_health_checks, \ - llm_router, \ llm_model_list, \ general_settings, \ proxy_budget_rescheduler_min_time, \ @@ -1707,7 +1707,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState raise ValueError("External offering mode requires an empty model_list and store_model_in_db disabled") if not await offerings_manager.reload(initial=True): raise ValueError("Initial external offering configuration is invalid") - llm_router = offerings_manager.router + llm_router = offerings_manager.router # rebind-ok: publish the process-wide router used by native proxy routes offerings_guard: Final = ( OfferingAccessGuard(offerings_manager.router, general_settings) if offerings_manager is not None else None ) From 04c0eb0e79c349888f7fed86bba4bdb19a10f9a6 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 8 Oct 2026 11:29:23 +0200 Subject: [PATCH 14/34] fix: validate model metadata at native type boundaries --- litellm/integrations/custom_logger.py | 2 +- litellm/llms/chatgpt/common_utils.py | 2 +- litellm/llms/chatgpt/model_info.py | 17 ++++-- litellm/llms/model_inventory.py | 5 +- litellm/llms/openai_like/model_info.py | 25 +++++--- .../proxy/common_utils/model_listing_utils.py | 2 +- litellm/proxy/offering_router.py | 50 ++++++++++----- litellm/proxy/proxy_server.py | 8 ++- litellm/router.py | 61 ++++++++++++++----- litellm/types/proxy/model_inventory.py | 22 ++++++- 10 files changed, 142 insertions(+), 52 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 45fce665bf0..6d3bfef2a2d 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -89,7 +89,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac turn_off_message_logging: bool = False, # deprecated param, use `turn_off_message_logging` instead message_logging: bool = True, - **kwargs, + **kwargs: object, # kwargs-ok: legacy logger initialization accepts and ignores extension options ) -> None: """ Args: diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index fe33219f110..9078e804114 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -229,7 +229,7 @@ def get_chatgpt_default_headers( access_token: str, account_id: str | None, session_id: str | None = None, -) -> dict: +) -> dict[str, str]: originator: Final = get_chatgpt_originator() user_agent: Final = get_chatgpt_user_agent(originator) headers: Final = { diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index 99e4a63ca7e..790a2e41d55 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -3,7 +3,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # typed native cache and HTTP boundaries import httpx from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator @@ -11,7 +11,12 @@ from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, fie from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS -from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelInventory, +) from litellm.types.proxy.model_metadata import GatewayModelMetadata from .authenticator import Authenticator @@ -169,6 +174,8 @@ async def get_chatgpt_model_inventory( authenticator: Authenticator | None = None, force_refresh: bool = False, ) -> SupplierModelInventory | SupplierInventoryUnavailable: + inventory_cache: Final = cast(ModelInventoryCache, cache) # cast-ok: native cache implements this protocol + inventory_client: Final = cast(ModelInventoryHTTPClient, client) # cast-ok: native HTTP handler implements get auth: Final = authenticator if authenticator is not None else Authenticator() try: trusted_base: Final = chatgpt_model_inventory_api_base(api_base, authenticator=auth) @@ -189,10 +196,10 @@ async def get_chatgpt_model_inventory( cache_key: Final = ( "chatgpt_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() ) - cached: Final[object] = cache.get_cache(cache_key) + cached: Final[object] = inventory_cache.get_cache(cache_key) if not force_refresh and isinstance(cached, SupplierModelInventory): return cached - response: Final = await client.get( + response: Final = await inventory_client.get( url=url, headers=headers, timeout=httpx.Timeout(5.0), @@ -211,7 +218,7 @@ async def get_chatgpt_model_inventory( inventory: Final = SupplierModelInventory( MappingProxyType({card.slug: card.metadata() for card in catalog.models}), credential_scope ) - cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) + inventory_cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) except httpx.HTTPStatusError: return SupplierInventoryUnavailable("http", credential_scope) except ValidationError: diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py index 2f065f823fe..ebef987a3bc 100644 --- a/litellm/llms/model_inventory.py +++ b/litellm/llms/model_inventory.py @@ -78,7 +78,8 @@ async def get_deployment_model_metadata( client: AsyncHTTPHandler, cache: InMemoryCache, ) -> Mapping[str, object]: - if params.get("use_clientside_credentials") or "*" in params.model: + metadata: Final = TypeAdapter(Mapping[str, object]).validate_python(params.model_dump()) + if metadata.get("use_clientside_credentials") or "*" in params.model: return _EMPTY_METADATA connection: Final = ( (params.model.removeprefix("chatgpt/"), "chatgpt", None, params.api_base) @@ -89,7 +90,7 @@ async def get_deployment_model_metadata( if provider != "chatgpt" and provider not in MODEL_INFO_DISCOVERY_PROVIDERS: return _EMPTY_METADATA api_key: Final = params.api_key or dynamic_api_key - headers: Final = _HEADERS.validate_python(params.get("extra_headers") or params.get("headers") or {}) + headers: Final = _HEADERS.validate_python(metadata.get("extra_headers") or metadata.get("headers") or {}) inventory: Final = await get_provider_model_inventory( provider=provider, api_base=api_base, diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 012d00c290f..1633d8c3c32 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -2,7 +2,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Annotated, Final, TypeAlias +from typing import Annotated, Final, TypeAlias, cast # noqa: TID251 # typed native cache and HTTP boundaries import httpx from pydantic import BeforeValidator, ConfigDict, ValidationError, ValidationInfo, field_validator @@ -10,7 +10,12 @@ from pydantic import BeforeValidator, ConfigDict, ValidationError, ValidationInf from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelInventory, +) from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.utils import add_path_to_api_base @@ -207,17 +212,19 @@ async def get_openai_compatible_model_inventory( cache: InMemoryCache, force_refresh: bool = False, ) -> SupplierModelInventory | SupplierInventoryUnavailable: + inventory_cache: Final = cast(ModelInventoryCache, cache) # cast-ok: native cache implements this protocol + inventory_client: Final = cast(ModelInventoryHTTPClient, client) # cast-ok: native HTTP handler implements get url: Final = add_path_to_api_base(api_base, "/v1/models") cache_key: Final = ( "upstream_model_info:" + hashlib.sha256(json.dumps((provider, url, sorted(headers.items()))).encode()).hexdigest() ) - cached: Final[object] = cache.get_cache(cache_key) + cached: Final[object] = inventory_cache.get_cache(cache_key) if not force_refresh and isinstance(cached, (SupplierModelInventory, SupplierInventoryUnavailable)): return cached try: - response: Final = await client.get( + response: Final = await inventory_client.get( url=url, headers=dict(headers), timeout=httpx.Timeout(5.0), @@ -233,23 +240,23 @@ async def get_openai_compatible_model_inventory( or len(frozenset(ids)) != len(ids) ): unavailable: Final = SupplierInventoryUnavailable("malformed") - cache.set_cache(cache_key, unavailable, ttl=60) + inventory_cache.set_cache(cache_key, unavailable, ttl=60) return unavailable except httpx.HTTPStatusError: http_failure: Final = SupplierInventoryUnavailable("http") - cache.set_cache(cache_key, http_failure, ttl=60) + inventory_cache.set_cache(cache_key, http_failure, ttl=60) return http_failure except ValidationError: malformed: Final = SupplierInventoryUnavailable("malformed") - cache.set_cache(cache_key, malformed, ttl=60) + inventory_cache.set_cache(cache_key, malformed, ttl=60) return malformed except Exception: # noqa: BLE001 # optional upstream discovery must not interrupt proxy refresh transport: Final = SupplierInventoryUnavailable("transport") - cache.set_cache(cache_key, transport, ttl=60) + inventory_cache.set_cache(cache_key, transport, ttl=60) return transport inventory: Final = SupplierModelInventory( MappingProxyType({card.id: card.token_limits(provider) for card in models.data}) ) - cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) + inventory_cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) return inventory diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 0c702ac4139..4c3d85fd4eb 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -349,7 +349,7 @@ class TeamModelNameTranslator: @staticmethod def resolve_public_name( model_id: str, - available_models: list[str], + available_models: Sequence[str], llm_router: Router | None, general_settings: Mapping[str, object], ) -> str: diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index 7998f8c8de1..f4694ce2fdc 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass +from types import MappingProxyType from typing import Final, cast # noqa: TID251 # native Router dynamic attribute dispatch from pydantic import TypeAdapter @@ -110,31 +111,50 @@ class OfferingAccessGuard(CustomLogger): def __init__(self, router: OfferingRouterView, general_settings: Mapping[str, object] | None = None) -> None: super().__init__() self.router = router - self.general_settings = general_settings if general_settings is not None else {} + self.general_settings: Mapping[str, object] = ( + general_settings if general_settings is not None else MappingProxyType({}) + ) def _offering_name(self, name: str, auth: UserAPIKeyAuth) -> str: snapshot: Final = self.router.serving_snapshot() + auth_metadata: Final = _METADATA_ADAPTER.validate_python(auth.model_dump()) caller_target: Final = ( alias_target( name, - caller_alias_maps(auth.aliases, auth.team_model_aliases, auth.team_id, auth.team_id), + caller_alias_maps( + _METADATA_ADAPTER.validate_python(auth_metadata.get("aliases") or MappingProxyType({})), + _METADATA_ADAPTER.validate_python(auth_metadata.get("team_model_aliases") or MappingProxyType({})), + auth.team_id, + auth.team_id, + ), snapshot.available_models | frozenset(snapshot.unavailable_models), ) or name ) target: Final = resolve_model_group_alias(snapshot.router.model_group_alias, caller_target) or caller_target - names: Final = [ - row["model_name"] - for row in (snapshot.router.get_model_list() or []) - if isinstance(row.get("model_name"), str) - and ( - auth.user_role == LitellmUserRoles.PROXY_ADMIN - or not isinstance(row.get("model_info"), Mapping) - or row["model_info"].get("team_id") in (None, auth.team_id) - ) - ] + rows: Final = TypeAdapter(tuple[Mapping[str, object], ...]).validate_python( + snapshot.router.get_model_list() or () + ) + names: Final = tuple(name for name in (self._visible_model_name(row, auth) for row in rows) if name is not None) return TeamModelNameTranslator.resolve_public_name(target, names, snapshot.router, self.general_settings) + @staticmethod + def _visible_model_name(row: Mapping[str, object], auth: UserAPIKeyAuth) -> str | None: + name: Final = row.get("model_name") + if not isinstance(name, str): + return None + raw_model_info: Final = row.get("model_info") + model_info: Final[Mapping[str, object]] = ( + _METADATA_ADAPTER.validate_python(raw_model_info) + if isinstance(raw_model_info, Mapping) + else MappingProxyType({}) + ) + return ( + name + if auth.user_role == LitellmUserRoles.PROXY_ADMIN or model_info.get("team_id") in (None, auth.team_id) + else None + ) + async def async_filter_listed_models( self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] ) -> Sequence[str]: @@ -207,8 +227,10 @@ class OfferingAccessGuard(CustomLogger): kwargs.get("litellm_metadata") or kwargs.get("metadata") or {} ) raw_model_info: Final = metadata.get("model_info") - model_info: Final = ( - _METADATA_ADAPTER.validate_python(raw_model_info) if isinstance(raw_model_info, Mapping) else {} + model_info: Final[Mapping[str, object]] = ( + _METADATA_ADAPTER.validate_python(raw_model_info) + if isinstance(raw_model_info, Mapping) + else MappingProxyType({}) ) deployment_id: Final = model_info.get("id") snapshot: Final = self.router.serving_snapshot() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2e744fe15b6..f1cf58154d0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1690,7 +1690,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState from litellm.proxy.model_offerings import ModelOfferingsManager from litellm.proxy.offering_router import OfferingAccessGuard - offerings_path: Final = TypeAdapter(str | None).validate_python( + offerings_path: Final = TypeAdapter[str | None](str | None).validate_python( cast(object, general_settings.get("model_offerings_path")), # cast-ok: validate untyped config strict=True, ) @@ -1709,7 +1709,11 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState raise ValueError("Initial external offering configuration is invalid") llm_router = offerings_manager.router # rebind-ok: publish the process-wide router used by native proxy routes offerings_guard: Final = ( - OfferingAccessGuard(offerings_manager.router, general_settings) if offerings_manager is not None else None + OfferingAccessGuard( + offerings_manager.router, TypeAdapter(Mapping[str, object]).validate_python(general_settings) + ) + if offerings_manager is not None + else None ) if offerings_guard is not None: litellm.logging_callback_manager.add_litellm_callback(offerings_guard) diff --git a/litellm/router.py b/litellm/router.py index b0aaf5cedc4..4f6d9098431 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2699,13 +2699,16 @@ class Router: deployment["litellm_params"], kwargs ) request_defaults: Final = model_request_defaults( - deployment.get("model_info", {}), selected_params, kwargs, "completion" + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(selected_params), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + "completion", ) litellm_params: Final = deployment_params_with_request_defaults( {key: value for key, value in selected_params.items() if key != "silent_model"}, request_defaults, deployment.get("model_info", {}), - kwargs, + _MODEL_INFO_ADAPTER.validate_python(kwargs), ) silent_model: Final = selected_params.get("silent_model") @@ -3877,13 +3880,16 @@ class Router: deployment["litellm_params"], kwargs ) request_defaults: Final = model_request_defaults( - deployment.get("model_info", {}), selected_params, kwargs, "completion" + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(selected_params), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + "completion", ) litellm_params: Final = deployment_params_with_request_defaults( {key: value for key, value in selected_params.items() if key != "silent_model"}, request_defaults, deployment.get("model_info", {}), - kwargs, + _MODEL_INFO_ADAPTER.validate_python(kwargs), ) silent_model: Final = selected_params.get("silent_model") @@ -5457,14 +5463,17 @@ class Router: request_defaults: Final = model_request_defaults( deployment.get("model_info", {}), - deployment["litellm_params"], - kwargs, + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + _MODEL_INFO_ADAPTER.validate_python(kwargs), getattr(original_generic_function, "__name__", ""), ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name=function_name) data: Final = deployment_params_with_request_defaults( - deployment["litellm_params"], request_defaults, deployment.get("model_info", {}), kwargs + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + request_defaults, + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(kwargs), ) model_name: Final = data["model"] self.total_calls[model_name] += 1 @@ -6146,12 +6155,18 @@ class Router: request_kwargs=kwargs, ) request_defaults: Final = model_request_defaults( - deployment.get("model_info", {}), deployment["litellm_params"], kwargs, handler_name + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + handler_name, ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="generic_api_call") data: Final = deployment_params_with_request_defaults( - deployment["litellm_params"], request_defaults, deployment.get("model_info", {}), kwargs + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + request_defaults, + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(kwargs), ) model_name: Final = data["model"] @@ -10813,15 +10828,21 @@ class Router: if not indices: return None - deployments: Final = tuple(self.model_list[index] for index in indices) + deployments: Final = tuple(_MODEL_INFO_ADAPTER.validate_python(self.model_list[index]) for index in indices) model_infos: Final = tuple( MappingProxyType( { - **self.get_discovered_model_info((deployment.get("model_info") or MappingProxyType({})).get("id")), + **self.get_discovered_model_info( + _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or MappingProxyType({})).get( + "id" + ) + ), **MappingProxyType( { k: v - for k, v in (deployment.get("model_info") or MappingProxyType({})).items() + for k, v in _MODEL_INFO_ADAPTER.validate_python( + deployment.get("model_info") or MappingProxyType({}) + ).items() if v is not None } ), @@ -10829,7 +10850,10 @@ class Router: ) for deployment in deployments ) - params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments) + params: Final = tuple( + _MODEL_INFO_ADAPTER.validate_python(deployment.get("litellm_params") or MappingProxyType({})) + for deployment in deployments + ) # base_model resolution mirrors get_router_model_info: unset or blank means the # deployment's own model name is the cost-map key. cost_map_keys: Final = tuple( @@ -10848,15 +10872,20 @@ class Router: max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), deployments=tuple( ModelListingDeployment( - cost_map_key=( - model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") - ), + cost_map_key=self._model_listing_cost_map_key(model_info, litellm_params), model_info=model_info, ) for model_info, litellm_params in zip(model_infos, params) ), ) + @staticmethod + def _model_listing_cost_map_key( + model_info: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> str | None: + key: Final = model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") + return key if isinstance(key, str) and key else None + @staticmethod def _widest_configured_limit(model_infos: Sequence[Mapping[str, object]], field: str) -> int | None: """The largest usable value of ``field`` across a group's configured model_info blocks.""" diff --git a/litellm/types/proxy/model_inventory.py b/litellm/types/proxy/model_inventory.py index b3ea0ae9ddf..17f41454f1d 100644 --- a/litellm/types/proxy/model_inventory.py +++ b/litellm/types/proxy/model_inventory.py @@ -1,6 +1,8 @@ from collections.abc import Mapping from dataclasses import dataclass -from typing import Literal +from typing import Literal, Protocol + +import httpx @dataclass(frozen=True, slots=True) @@ -13,3 +15,21 @@ class SupplierModelInventory: class SupplierInventoryUnavailable: reason: Literal["authentication", "http", "transport", "malformed"] credential_scope: str | None = None + + +class ModelInventoryCache(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: object, *, ttl: int) -> None: ... + + +class ModelInventoryHTTPClient(Protocol): + async def get( + self, + *, + url: str, + headers: Mapping[str, str], + timeout: httpx.Timeout, + follow_redirects: bool, + max_response_bytes: int, + ) -> httpx.Response: ... From 127f6de0d3d2a4852dd497a81e3694153e958689 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 8 Oct 2026 11:48:29 +0200 Subject: [PATCH 15/34] fix: narrow legacy metadata lookup inputs --- litellm/llms/chatgpt/responses/transformation.py | 5 ++++- litellm/router.py | 14 +++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 7163cc5d3e0..36d171ba75b 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -112,7 +112,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): } filtered: Final = {k: v for k, v in request.items() if k in allowed_keys} - service_tier: Final = _CHATGPT_SERVICE_TIERS.get(request.get("service_tier")) + requested_service_tier: Final[object] = request.get("service_tier") + service_tier: Final = ( + _CHATGPT_SERVICE_TIERS.get(requested_service_tier) if isinstance(requested_service_tier, str) else None + ) if service_tier is not None: filtered["service_tier"] = service_tier return filtered diff --git a/litellm/router.py b/litellm/router.py index 4f6d9098431..375e70d1b2d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9878,7 +9878,9 @@ class Router: - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values - ValueError: If supported_environments is not set in model_info or not one of the valid values """ - return model_info_is_active_for_environment(model_info=deployment.model_info) + return model_info_is_active_for_environment( + model_info=_MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True)) + ) def set_model_list(self, model_list: list): original_model_list: Final = copy.deepcopy(model_list) @@ -10794,13 +10796,11 @@ class Router: cache=self.cache.in_memory_cache, ) - def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, object]: + def get_discovered_model_info(self, model_id: object) -> Mapping[str, object]: + if not isinstance(model_id, str): + return MappingProxyType({}) cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) - if ( - model_id is not None - and isinstance(cached, DiscoveredDeploymentModelInfo) - and cached.deployment is self.get_model_info(model_id) - ): + if isinstance(cached, DiscoveredDeploymentModelInfo) and cached.deployment is self.get_model_info(model_id): configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) return MappingProxyType({}) From a0563b91f429efff1918f7e6f5bff252c82a8e18 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 8 Oct 2026 13:30:42 +0200 Subject: [PATCH 16/34] fix(proxy): enforce offering batch authority and validate supplier metadata --- examples/model_offerings/README.md | 2 + litellm/llms/chatgpt/chat/transformation.py | 2 +- litellm/llms/chatgpt/model_info.py | 4 + litellm/proxy/offering_router.py | 8 ++ tests/test_litellm_rust/test_traces.py | 3 +- .../llms/chatgpt/chat/test_transformation.py | 9 +- tests/unit/llms/chatgpt/test_model_info.py | 135 ++++++++++++++++-- tests/unit/proxy/test_model_offerings.py | 79 +++++++++- tests/unit/proxy/test_offering_router.py | 77 +++++++++- .../unit/test_router_model_cost_isolation.py | 20 ++- .../unit/types/proxy/test_model_offerings.py | 8 ++ 11 files changed, 324 insertions(+), 23 deletions(-) diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index 24117b4c5b0..f48cfb8a26c 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -68,4 +68,6 @@ Connection identities use HMAC with the existing private server salt/master key. Caller `extra_body.model` overrides are rejected before supplier forwarding; harmless extra body fields remain supported. Structured `reasoning_effort` objects such as `{effort: high, summary: auto}` select the same configured output budget as the corresponding effort string. +Native provider batch creation and uploads with purpose `batch` are rejected in external offering mode. Native batch records can select their own supplier models, which would bypass the configured offering catalog. Existing proxy configurations without external offerings retain native batch support. + The optional bundled cost-map JSON-schema extension is deferred: its trusted-base guard validates PR artifacts with the base generator. The runtime metadata/configuration contract validates separate total/input/output limits and defaults independently, and unknown optional bundled catalog properties remain forward compatible. diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index b3c28e94a77..7c985b7dd5e 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -33,7 +33,7 @@ class ChatGPTConfig(OpenAIConfig): api_key: str | None, custom_llm_provider: str, ) -> tuple[str | None, str | None, str]: - dynamic_api_base: Final = api_base or self.api_base_without_login() + dynamic_api_base: Final = self.api_base_without_login() try: dynamic_api_key: Final = self.authenticator.get_access_token() except GetAccessTokenError as e: diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index 790a2e41d55..2a3707de87e 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -55,6 +55,8 @@ class _ChatGPTModel(GatewayModelMetadata): cls.validate_field_input( "context_window" if info.field_name == "max_context_window" else info.field_name, normalized ) + if info.field_name == "supports_parallel_tool_calls": + cls.validate_field_input("supports_parallel_function_calling", normalized) return normalized @field_validator("max_context_window", mode="before") @@ -85,6 +87,8 @@ class _ChatGPTModel(GatewayModelMetadata): if native_default == "ultra" and "max" in (api_efforts or ()) else api_efforts[-1] if native_default == "ultra" and api_efforts + else None + if native_default == "ultra" else "disabled" if native_default == "persistent" else native_default diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index f4694ce2fdc..e5961cd3fb3 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -170,6 +170,14 @@ class OfferingAccessGuard(CustomLogger): data: Mapping[str, object], call_type: CallTypesLiteral, ) -> Exception | None: + if call_type in ("acreate_batch", "create_batch") or ( + call_type in ("acreate_file", "create_file") and data.get("purpose") == "batch" + ): + return litellm.BadRequestError( + message="Native provider batches are unavailable in external offering mode because batch records cannot enforce offering selection", + model="", + llm_provider="", + ) extra_body: Final = data.get("extra_body") if isinstance(extra_body, Mapping) and "model" in extra_body: return litellm.BadRequestError( diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index f07bc776bce..47238070f40 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -174,10 +174,11 @@ async def test_schema_setup_uses_configured_retention(recording_server: Recordin request.raw_body for request in recording_server.requests if b"MODIFY TTL" in request.raw_body ) assert all(b"INTERVAL 7 DAY" in statement for statement in ttl_statements) - assert tuple(request.raw_body.strip() for request in recording_server.requests[-3:]) == ( + assert tuple(statement.strip() for statement in ttl_statements[-4:]) == ( b"ALTER TABLE `trace_test`.otel_traces MODIFY TTL toDateTime(Timestamp) + INTERVAL 7 DAY", b"ALTER TABLE `trace_test`.agent_traces_by_key MODIFY TTL toDateTime(StartTs) + INTERVAL 7 DAY", b"ALTER TABLE `trace_test`.spend_logs MODIFY TTL toDateTime(start_time) + INTERVAL 7 DAY", + b"ALTER TABLE `trace_test`.lens_feedback MODIFY TTL toDateTime(CreatedAt) + INTERVAL 7 DAY", ) diff --git a/tests/unit/llms/chatgpt/chat/test_transformation.py b/tests/unit/llms/chatgpt/chat/test_transformation.py index 261936b2656..bfa99cc64ce 100644 --- a/tests/unit/llms/chatgpt/chat/test_transformation.py +++ b/tests/unit/llms/chatgpt/chat/test_transformation.py @@ -2,11 +2,16 @@ import json from pathlib import Path from typing import Final +import pytest + from litellm.llms.chatgpt.authenticator import prevent_device_login from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig -def test_configured_native_endpoint_is_used_with_saved_authorization(tmp_path: Path) -> None: +def test_configured_native_endpoint_is_used_with_saved_authorization( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_API_BASE", "https://configured.test/native") auth_file: Final = tmp_path / "auth.json" auth_file.write_text(json.dumps({"access_token": "local-fixture-token", "expires_at": 9999999999})) config: Final = ChatGPTConfig() @@ -14,7 +19,7 @@ def test_configured_native_endpoint_is_used_with_saved_authorization(tmp_path: P with prevent_device_login(): result: Final = config._get_openai_compatible_provider_info( model="local-fixture-model", - api_base="https://configured.test/native", + api_base="https://untrusted.test/capture", api_key=None, custom_llm_provider="chatgpt", ) diff --git a/tests/unit/llms/chatgpt/test_model_info.py b/tests/unit/llms/chatgpt/test_model_info.py index e49f1f744a8..f4c4dbc1454 100644 --- a/tests/unit/llms/chatgpt/test_model_info.py +++ b/tests/unit/llms/chatgpt/test_model_info.py @@ -8,8 +8,127 @@ import pytest from litellm.caching.in_memory_cache import InMemoryCache from litellm.llms.chatgpt.authenticator import Authenticator from litellm.llms.chatgpt.common_utils import GetAccessTokenError -from litellm.llms.chatgpt.model_info import _ChatGPTModel, get_chatgpt_model_info +from litellm.llms.chatgpt.model_info import ( + _ChatGPTModel, + chatgpt_model_inventory_identity, + get_chatgpt_model_info, + get_chatgpt_model_inventory, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory + + +@pytest.mark.parametrize( + "invalid", + ( + {"error": "denied", "models": []}, + {"models": [{"slug": " duplicate "}]}, + {"models": [{"slug": ""}]}, + {"models": [{"slug": "same"}, {"slug": "same"}]}, + {"models": [{"slug": "fixture", "context_window": "invalid"}]}, + {"models": [{"slug": "fixture", "supports_parallel_tool_calls": "false"}]}, + ), +) +async def test_malformed_refresh_never_replaces_a_successful_native_inventory( + invalid: dict[str, object], tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + (tmp_path / "auth.json").write_text( + json.dumps({"access_token": "fixture-token", "account_id": "fixture-account", "expires_at": 10**30}) + ) + responses: Final = iter(({"models": [{"slug": "fixture", "context_window": 0}]}, invalid, {"models": []})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _: httpx.Response(200, json=next(responses))) + ) as client: + handler.client = client + cache: Final = InMemoryCache() + first: Final = await get_chatgpt_model_inventory(client=handler, cache=cache) + assert isinstance(first, SupplierModelInventory) + assert set(first.models) == {"fixture"} + assert "codex_context_window" not in first.models["fixture"] + failed: Final = await get_chatgpt_model_inventory(client=handler, cache=cache, force_refresh=True) + assert isinstance(failed, SupplierInventoryUnavailable) + assert failed.reason == "malformed" + assert failed.credential_scope == first.credential_scope + assert await get_chatgpt_model_inventory(client=handler, cache=cache) is first + empty: Final = await get_chatgpt_model_inventory(client=handler, cache=cache, force_refresh=True) + assert isinstance(empty, SupplierModelInventory) + assert not empty.models + assert await get_chatgpt_model_inventory(client=handler, cache=cache) is empty + + +async def test_native_transport_failure_is_unavailable_and_never_an_empty_inventory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + (tmp_path / "auth.json").write_text(json.dumps({"access_token": "fixture-token", "expires_at": 10**30})) + + def timeout(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("fixture timeout", request=request) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(timeout)) as client: + handler.client = client + result: Final = await get_chatgpt_model_inventory(client=handler, cache=InMemoryCache()) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == "transport" + assert result.credential_scope is None + + +@pytest.mark.parametrize("content", (b"not valid json", b"\xff")) +def test_unreadable_native_account_does_not_claim_a_credential_change( + content: bytes, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_bytes(content) + assert chatgpt_model_inventory_identity() is None + + +def test_native_explicit_capabilities_override_native_defaults_without_conflating_context() -> None: + metadata: Final = _ChatGPTModel.model_validate( + { + "slug": "fixture", + "context_window": 900, + "max_context_window": 0, + "max_input_tokens": 600, + "max_output_tokens": 300, + "supports_function_calling": True, + "supports_parallel_function_calling": False, + "supports_parallel_tool_calls": True, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "ultra"], + "default_reasoning_effort": "ultra", + "supported_modalities": ["text"], + "input_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + } + ).metadata() + assert metadata["default_reasoning_effort"] == "high" + assert metadata["reasoning_effort_levels"] == ("low", "high") + assert metadata["supports_reasoning"] is True + assert metadata["supports_parallel_function_calling"] is False + assert metadata["supported_modalities"] == ("text",) + assert metadata["supported_output_modalities"] == ("text",) + assert metadata["max_input_tokens"] == 600 + assert metadata["max_output_tokens"] == 300 + assert metadata["codex_context_window"] == 900 + assert "context_window" not in metadata + assert "codex_max_context_window" not in metadata + + +@pytest.mark.parametrize("levels", (None, [])) +def test_unresolvable_ui_default_never_advertises_an_unsupported_api_effort( + levels: list[dict[str, str]] | None, +) -> None: + metadata: Final = _ChatGPTModel.model_validate( + {"slug": "fixture", "default_reasoning_level": "ultra", "supported_reasoning_levels": levels} + ).metadata() + assert "default_reasoning_effort" not in metadata async def test_account_catalog_preserves_client_policy_separately_from_model_limits( @@ -172,15 +291,12 @@ def test_codex_ui_efforts_are_not_advertised_as_api_efforts( assert metadata["default_reasoning_effort"] == expected -async def test_caller_api_base_cannot_exfiltrate_saved_server_oauth(monkeypatch: pytest.MonkeyPatch) -> None: - from unittest.mock import Mock - - from litellm.llms.chatgpt.model_info import get_chatgpt_model_inventory - from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable - +async def test_caller_api_base_cannot_exfiltrate_saved_server_oauth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setenv("CHATGPT_API_BASE", "https://trusted.test/backend-api/codex") - token: Final = Mock(side_effect=AssertionError("Rejected destinations must not even read the access token")) - monkeypatch.setattr(Authenticator, "get_access_token", token) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_text(json.dumps({"access_token": "fixture-server-token", "expires_at": 10**30})) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -194,4 +310,3 @@ async def test_caller_api_base_cannot_exfiltrate_saved_server_oauth(monkeypatch: ) assert isinstance(result, SupplierInventoryUnavailable) assert result.reason == "authentication" - token.assert_not_called() diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 6b90b459b95..a1fdeae9835 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from typing import Final @@ -87,6 +88,67 @@ async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload assert manager.router.model_names == {"manual"} +async def test_disabled_manual_offering_and_invalid_secrets_keep_authority_atomic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm.caching.caching import DualCache + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.offering_router import OfferingAccessGuard + + path: Final = tmp_path / "offerings.yaml" + monkeypatch.setenv("FIXTURE_OFFERING_KEY", "fixture-secret") + config: Final = { + "version": 1, + "providers": { + "supplier": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "api_key": "os.environ/FIXTURE_OFFERING_KEY", + "headers": {"X-Session": "os.environ/FIXTURE_OFFERING_KEY"}, + } + }, + "offerings": [ + {"model_name": "manual", "source": "manual", "provider": "supplier", "upstream_model": "backend"} + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + active: Final = manager.router.latest_snapshot() + assert active.available_models == {"manual"} + assert await manager.reload() + assert manager.router.latest_snapshot() is active + monkeypatch.delenv("FIXTURE_OFFERING_KEY") + assert not await manager.reload() + assert manager.router.latest_snapshot() is active + assert manager.last_error == "ValueError" + path.write_bytes(b" " * (1024 * 1024 + 1)) + assert not await manager.reload() + assert manager.router.latest_snapshot() is active + assert manager.last_error == "ConfigTooLarge" + monkeypatch.setenv("FIXTURE_OFFERING_KEY", "fixture-secret") + path.write_text(yaml.safe_dump({**config, "offerings": [{**config["offerings"][0], "enabled": False}]})) + assert await manager.reload() + assert not manager.router.latest_snapshot().available_models + assert "disabled" in manager.router.latest_snapshot().unavailable_models["manual"] + rejected: Final = await OfferingAccessGuard(manager.router).async_pre_call_hook( + UserAPIKeyAuth(), DualCache(), {"model": "manual"}, "acompletion" + ) + assert isinstance(rejected, litellm.ServiceUnavailableError) + path.write_text(yaml.safe_dump(config)) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == {"manual"} + + async def test_new_connection_cannot_reuse_another_credentials_inventory(tmp_path: Path) -> None: path: Final = tmp_path / "offerings.yaml" first_config: Final = { @@ -442,13 +504,11 @@ async def test_private_connection_identity_is_stable_and_changes_on_credentials( async def test_native_account_change_changes_private_deployment_identity( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - from litellm.llms.chatgpt.authenticator import Authenticator - monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") monkeypatch.setenv("CHATGPT_API_BASE", "https://native.test/backend-api/codex") - account: Final = {"id": "first"} - monkeypatch.setattr(Authenticator, "get_account_id", lambda self: account["id"]) - monkeypatch.setattr(Authenticator, "get_access_token", lambda self, **kwargs: "fixture-oauth") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text(json.dumps({"access_token": "fixture-oauth", "account_id": "first", "expires_at": 10**30})) path: Final = tmp_path / "offerings.yaml" path.write_text( yaml.safe_dump( @@ -463,15 +523,20 @@ async def test_native_account_change_changes_private_deployment_identity( ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() + responses: Final = iter((200, 401, 200)) async with httpx.AsyncClient( - transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"models": [{"slug": "fixture"}]})) + transport=httpx.MockTransport( + lambda request: httpx.Response(next(responses), json={"models": [{"slug": "fixture"}]}) + ) ) as client: handler.client = client manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) assert await manager.reload(initial=True) first: Final = manager.router.get_model_ids() assert len(first) == 1 - account["id"] = "second" + auth_file.write_text(json.dumps({"access_token": "second-oauth", "account_id": "second", "expires_at": 10**30})) + assert await manager.reload(force_inventory=True) + assert not manager.router.latest_snapshot().available_models assert await manager.reload(force_inventory=True) assert manager.router.get_model_ids() != first assert manager.router.latest_snapshot().available_models == {"selected"} diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 575da05b2e5..31f6f228455 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -2,6 +2,7 @@ from types import MappingProxyType from typing import Final import pytest +from starlette.types import Message, Receive, Scope, Send import litellm from litellm.caching.caching import DualCache @@ -13,11 +14,85 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_checks import can_customer_access_model, can_key_call_model -from litellm.proxy.offering_router import OfferingAccessGuard, OfferingRouterView, OfferingServingSnapshot +from litellm.proxy.offering_router import ( + OfferingAccessGuard, + OfferingRouterView, + OfferingServingSnapshot, + OfferingSnapshotMiddleware, +) from litellm.router import Router +from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes +async def test_asgi_stream_pins_authority_until_final_chunk_and_passes_lifespan_through() -> None: + first: Final = OfferingServingSnapshot( + Router(model_list=[]), frozenset({"first"}), MappingProxyType({}), frozenset() + ) + second: Final = OfferingServingSnapshot( + Router(model_list=[]), frozenset({"second"}), MappingProxyType({}), frozenset() + ) + view: Final = OfferingRouterView(first) + sent: Final[list[Message]] = [] + + async def receive() -> Message: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + sent.append(message) + + async def stream(scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "lifespan": + assert view.serving_snapshot() is second + await send({"type": "lifespan.startup.complete"}) + return + assert view.serving_snapshot() is first + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"first", "more_body": True}) + view.publish_snapshot(second) + assert view.serving_snapshot() is first + assert view.latest_snapshot() is second + await send({"type": "http.response.body", "body": b"last", "more_body": False}) + + middleware: Final = OfferingSnapshotMiddleware(stream, lambda: view) + await middleware({"type": "http"}, receive, send) + assert view.serving_snapshot() is second + assert tuple(message["body"] for message in sent if message["type"] == "http.response.body") == (b"first", b"last") + await middleware({"type": "lifespan"}, receive, send) + assert sent[-1] == {"type": "lifespan.startup.complete"} + + +def test_native_mutations_cannot_publish_unmanaged_routes_in_external_mode() -> None: + native: Final = Router(model_list=[]) + view: Final = OfferingRouterView(OfferingServingSnapshot(native, frozenset(), MappingProxyType({}), frozenset())) + with pytest.raises(litellm.BadRequestError, match="external offering configuration"): + view.add_deployment(Deployment(model_name="unmanaged", litellm_params=LiteLLM_Params(model="openai/unmanaged"))) + assert not native.get_model_list() + view.num_retries = 2 + assert native.num_retries == 2 + + +async def test_native_batches_cannot_bypass_offering_selection_with_uploaded_record_models() -> None: + view: Final = OfferingRouterView( + OfferingServingSnapshot(Router(model_list=[]), frozenset({"selected"}), MappingProxyType({}), frozenset()) + ) + guard: Final = OfferingAccessGuard(view) + auth: Final = UserAPIKeyAuth() + cache: Final = DualCache() + for data in ({"model": "selected"}, {"input_file_id": "file-already-uploaded"}): + rejected: Final = await guard.async_pre_call_hook(auth, cache, data, "acreate_batch") + assert isinstance(rejected, litellm.BadRequestError) + assert "batch records" in rejected.message + assert isinstance( + await guard.async_pre_call_hook(auth, cache, {"model": "selected", "purpose": "batch"}, "acreate_file"), + litellm.BadRequestError, + ) + assert ( + await guard.async_pre_call_hook(auth, cache, {"model": "selected", "purpose": "assistants"}, "acreate_file") + is None + ) + + async def test_unavailable_and_unselected_offerings_cannot_forward_or_override_connection() -> None: router: Final = OfferingRouterView( OfferingServingSnapshot( diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 66365e0c3a3..29062f52c85 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -12,6 +12,7 @@ import copy import logging import os import re +from collections.abc import Mapping from typing import Final from unittest.mock import Mock, patch @@ -22,8 +23,8 @@ import litellm from litellm import Router from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE -from litellm.litellm_core_utils.ptu_pricing import ptu_config_error from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.ptu_pricing import ptu_config_error from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -33,6 +34,23 @@ from litellm.utils import ( ) +@pytest.mark.parametrize( + ("info", "params", "expected"), + ( + ({"base_model": "info-base"}, {"base_model": "params-base", "model": "backend"}, "info-base"), + ({}, {"base_model": "params-base", "model": "backend"}, "params-base"), + ({"base_model": ""}, {"model": "backend"}, "backend"), + ({}, {"model": ""}, None), + ({}, {}, None), + ({"base_model": 42}, {"model": "backend"}, None), + ), +) +def test_model_listing_cost_map_key_preserves_explicit_base_model_precedence( + info: Mapping[str, object], params: Mapping[str, object], expected: str | None +) -> None: + assert Router._model_listing_cost_map_key(info, params) == expected + + def _simulate_price_data_reload(fetched_catalog): """Drive what a price data reload does to this process's litellm state. diff --git a/tests/unit/types/proxy/test_model_offerings.py b/tests/unit/types/proxy/test_model_offerings.py index 11270c495d1..b0999826c1a 100644 --- a/tests/unit/types/proxy/test_model_offerings.py +++ b/tests/unit/types/proxy/test_model_offerings.py @@ -30,6 +30,14 @@ from litellm.types.proxy.model_offerings import ModelOfferingsConfig ] }, {"providers": {"supplier": {"provider": "chatgpt", "api_key": "ignored-key"}}}, + {"providers": {"supplier": {"provider": "openai"}}}, + {"providers": {" supplier ": {"provider": "openrouter"}}}, + {"offerings": [{"model_name": "*", "source": "manual", "provider": "supplier", "upstream_model": "backend"}]}, + { + "offerings": [ + {"model_name": "selected", "source": "manual", "provider": "missing", "upstream_model": "backend"} + ] + }, ), ) def test_invalid_operator_edits_are_rejected(change) -> None: From c9dea8f039d4d31f8be42ddd299fa943dd9f8f52 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 8 Oct 2026 13:53:04 +0200 Subject: [PATCH 17/34] test(llms): use isolated native account fixture --- tests/unit/llms/test_model_inventory.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/llms/test_model_inventory.py b/tests/unit/llms/test_model_inventory.py index 8b0a7ed5ef6..336824f7705 100644 --- a/tests/unit/llms/test_model_inventory.py +++ b/tests/unit/llms/test_model_inventory.py @@ -1,8 +1,9 @@ +import json +from pathlib import Path from typing import Final import pytest -from litellm.llms.chatgpt.authenticator import Authenticator from litellm.llms.model_inventory import model_inventory_api_base, model_inventory_identity @@ -20,10 +21,14 @@ def test_inventory_connection_defaults_and_explicit_endpoints(provider, expected assert model_inventory_identity(provider) is None -def test_native_inventory_registry_uses_server_owned_backend_and_identity(monkeypatch: pytest.MonkeyPatch) -> None: +def test_native_inventory_registry_uses_server_owned_backend_and_identity( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: trusted: Final = "https://server-owned.test/backend-api/codex" monkeypatch.setenv("CHATGPT_API_BASE", trusted) - monkeypatch.setattr(Authenticator, "get_account_id", lambda self: "fixture-account") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "auth.json") + (tmp_path / "auth.json").write_text(json.dumps({"account_id": "fixture-account"})) assert model_inventory_api_base("chatgpt", None) == trusted assert model_inventory_api_base("chatgpt", trusted + "/") == trusted assert model_inventory_identity("chatgpt") == "fixture-account" From c40f4c82be90cbdaa2cdd44f4a5ce810c5759519 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 8 Oct 2026 14:19:19 +0200 Subject: [PATCH 18/34] ci(rust): bound feature-matrix build artifacts --- .github/workflows/test-rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index c07487d5b49..c483930f769 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -98,6 +98,11 @@ jobs: rust-test: runs-on: ubuntu-latest timeout-minutes: 30 + env: + # Feature combinations retain many build variants; keep runner disk usage bounded. + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + CARGO_INCREMENTAL: "0" defaults: run: working-directory: litellm-rust From f3fbcdeafce3a9b700af3ecbdad5529a187a3cc2 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 12 Oct 2026 14:26:42 +0200 Subject: [PATCH 19/34] feat(proxy): enforce offering-owned provider endpoint policies --- examples/model_offerings/README.md | 31 ++ litellm/proxy/model_offerings.py | 57 ++- litellm/proxy/offering_router.py | 108 +++++- litellm/types/proxy/model_offerings.py | 38 ++ tests/unit/proxy/test_model_offerings.py | 358 ++++++++++++++++++ tests/unit/proxy/test_offering_router.py | 100 +++++ .../unit/types/proxy/test_model_offerings.py | 44 ++- 7 files changed, 710 insertions(+), 26 deletions(-) diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index f48cfb8a26c..b05e3e9b9cd 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -68,6 +68,37 @@ Connection identities use HMAC with the existing private server salt/master key. Caller `extra_body.model` overrides are rejected before supplier forwarding; harmless extra body fields remain supported. Structured `reasoning_effort` objects such as `{effort: high, summary: auto}` select the same configured output budget as the corresponding effort string. +## Provider endpoint policy + +An offering can own a typed `provider_policy`. Currently OpenRouter chat offerings support an exact endpoint allowlist with fallbacks disabled. The policy belongs to the offering, not the connection, and never accepts credentials or arbitrary request-body options. The supplier must match the policy provider and `model_info.mode` must explicitly be `chat` + +```yaml +offerings: + - model_name: selected-endpoint + source: auto + provider: openrouter + upstream_model: organization/model + enabled: false + provider_policy: + provider: openrouter + only: [supplier/exact-endpoint] + allow_fallbacks: false + model_info: + mode: chat + context_window: 32000 + max_output_tokens: 4000 +``` + +Append this entry to the external file's existing `offerings`, using its configured OpenRouter connection name. The example limits are synthetic. Verify endpoint-specific values before enabling it. `only` contains provider slugs, not URLs, and a single full slug pins one endpoint. Multiple explicit slugs intentionally allow any endpoint in that list; `allow_fallbacks` must remain false + +The gateway merges its policy into harmless caller `extra_body` fields and verifies it again at deployment dispatch. Caller `provider`, `models`, `route`, fallback controls, nested body overrides and mock responses are rejected for policy-controlled offerings, even through aliases. Caller deployment selectors are rejected throughout external mode. Policy-controlled requests disable LiteLLM model-group fallbacks as well as OpenRouter provider fallback. Unavailable endpoints fail instead of serving a different offering. Retries may still contact the same pinned endpoint + +Policy-controlled requests disable response-cache reads and writes, overriding caller cache settings. Existing response-cache keys do not reliably separate provider endpoints; cached output from another endpoint must not satisfy a pinned request + +Chat Completions, native Responses and translated Anthropic Messages preserve the policy, including streaming. Other APIs are rejected for policy-controlled offerings. Batch execution remains unavailable throughout external mode. Offerings without policy retain existing provider-routing behavior, apart from the deployment-selector restriction + +AUTO still checks model membership, but a pinned endpoint does not inherit aggregate supplier or bundled model metadata. Only explicit `model_info` is advertised, with unknown limits omitted. Endpoint inventory polling is not added: verify and maintain those overrides separately. Policy edits change that offering's private deployment ID and publish atomically; requests already streaming retain their previous snapshot + Native provider batch creation and uploads with purpose `batch` are rejected in external offering mode. Native batch records can select their own supplier models, which would bypass the configured offering catalog. Existing proxy configurations without external offerings retain native batch support. The optional bundled cost-map JSON-schema extension is deferred: its trusted-base guard validates PR artifacts with the base generator. The runtime metadata/configuration contract validates separate total/input/output limits and defaults independently, and unknown optional bundled catalog properties remain forward compatible. diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index 9794539e527..4b17e0773d1 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -140,19 +140,38 @@ def _unavailable_reason(offering: ModelOffering, state: _ProviderState) -> str | return None +def _deployment_id(offering: ModelOffering, connection: _ResolvedConnection) -> str: + identity: Final = ( + offering.model_name, + connection.fingerprint, + offering.upstream_model, + connection.supplier_identity, + ) + return ( + "offering-" + + hashlib.sha256( + json.dumps( + identity + (offering.provider_policy.model_dump(mode="json"),) + if offering.provider_policy is not None + else identity + ).encode() + ).hexdigest() + ) + + def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, object]: connection: Final = state.connection reason: Final = _unavailable_reason(offering, state) supplier_info: Final[Mapping[str, object]] = ( state.inventory.models.get(offering.upstream_model, {}) - if offering.source == "auto" and state.inventory is not None + if offering.source == "auto" and state.inventory is not None and offering.provider_policy is None else {} ) catalog: Final = GetModelCostMap.loaded_model_cost_map() bundled_info: Final = ( catalog.get(f"{connection.provider}/{offering.upstream_model}") or (catalog.get(offering.upstream_model, {}) if connection.provider == "openai" else {}) - if offering.source == "auto" + if offering.source == "auto" and offering.provider_policy is None else {} ) bundled_metadata: Final = GatewayModelMetadata.model_validate(bundled_info).model_dump( @@ -165,19 +184,17 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, **supplier_info, **offering.model_info.model_dump(mode="json", exclude_none=True), } - deployment_id: Final = ( - "offering-" - + hashlib.sha256( - json.dumps( - (offering.model_name, connection.fingerprint, offering.upstream_model, connection.supplier_identity) - ).encode() - ).hexdigest() - ) + deployment_id: Final = _deployment_id(offering, connection) return MappingProxyType( { "model_name": offering.model_name, "litellm_params": { "model": f"{connection.provider}/{offering.upstream_model}", + **( + {"extra_body": {"provider": dict(offering.provider_policy.request_body())}} + if offering.provider_policy is not None + else {} + ), **({"api_base": connection.api_base} if connection.api_base is not None else {}), **({"api_key": connection.api_key} if connection.api_key is not None else {}), **( @@ -293,7 +310,25 @@ class ModelOfferingsManager: if offering.model_name in available ) snapshot: Final = OfferingServingSnapshot( - candidate, available, unavailable, allowed_deployments, frozenset(candidate.get_model_ids()) + candidate, + available, + unavailable, + allowed_deployments, + frozenset(candidate.get_model_ids()), + MappingProxyType( + { + offering.model_name: offering.provider_policy + for offering in loaded.config.offerings + if offering.model_name in available and offering.provider_policy is not None + } + ), + MappingProxyType( + { + _deployment_id(offering, providers[offering.provider].connection): offering.provider_policy + for offering in loaded.config.offerings + if offering.model_name in available and offering.provider_policy is not None + } + ), ) self.config = loaded.config self.config_fingerprint = loaded.fingerprint diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index e5961cd3fb3..ed769238ee0 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -1,7 +1,7 @@ from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass +from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, cast # noqa: TID251 # native Router dynamic attribute dispatch @@ -16,6 +16,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator, alias_target, caller_alias_maps from litellm.router import Router from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.types.proxy.model_offerings import OpenRouterProviderPolicy from litellm.types.utils import CallTypes, CallTypesLiteral @@ -26,6 +27,8 @@ class OfferingServingSnapshot: unavailable_models: Mapping[str, str] allowed_deployments: frozenset[str] allowed_deployment_ids: frozenset[str] = frozenset() + provider_policies: Mapping[str, OpenRouterProviderPolicy] = field(default_factory=lambda: MappingProxyType({})) + deployment_policies: Mapping[str, OpenRouterProviderPolicy] = field(default_factory=lambda: MappingProxyType({})) _PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] = ContextVar( @@ -34,6 +37,35 @@ _PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] _MODEL_MUTATIONS: Final = frozenset(("set_model_list", "add_deployment", "upsert_deployment", "delete_deployment")) _VIEW_MEMBERS: Final = frozenset(("publish_snapshot", "pin_snapshot", "serving_snapshot", "latest_snapshot")) _METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_POLICY_CALLS: Final = frozenset(("acompletion", "completion", "aresponses", "responses", "anthropic_messages")) +_ROUTING_OVERRIDES: Final = frozenset( + ( + "provider", + "models", + "route", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "model_list", + "deployment_id", + "specific_deployment", + "model_id", + "disable_fallbacks", + "mock_response", + "mock_tool_calls", + "mock_timeout", + "mock_provider_error", + ) +) + + +def _policy_override(data: Mapping[str, object]) -> bool: + extra_body: Final = data.get("extra_body") + return bool(_ROUTING_OVERRIDES.intersection(data)) or ( + isinstance(extra_body, Mapping) + and bool((_ROUTING_OVERRIDES | frozenset(("model", "extra_body"))).intersection(extra_body)) + ) class OfferingRouterView(Router): @@ -169,7 +201,7 @@ class OfferingAccessGuard(CustomLogger): cache: DualCache, data: Mapping[str, object], call_type: CallTypesLiteral, - ) -> Exception | None: + ) -> dict[str, object] | Exception | None: # mutable-ok: CustomLogger requires a fresh dict for request replacement if call_type in ("acreate_batch", "create_batch") or ( call_type in ("acreate_file", "create_file") and data.get("purpose") == "batch" ): @@ -196,6 +228,7 @@ class OfferingAccessGuard(CustomLogger): "extra_headers", "headers", "provider_specific_header", + "client", ) ): return litellm.BadRequestError( @@ -203,13 +236,35 @@ class OfferingAccessGuard(CustomLogger): model="", llm_provider="", ) + if any(name in data for name in ("specific_deployment", "deployment_id", "model_id", "model_list")): + return litellm.BadRequestError( + message="External offering mode does not accept caller-defined deployment selection", + model="", + llm_provider="", + ) model: Final = data.get("model") if not isinstance(model, str): return None snapshot: Final = self.router.serving_snapshot() offering: Final = self._offering_name(model, user_api_key_dict) if offering in snapshot.available_models: - return None + policy: Final = snapshot.provider_policies.get(offering) + if policy is None: + return None + if call_type not in _POLICY_CALLS or _policy_override(data): + return litellm.BadRequestError( + message="The offering provider policy does not accept caller routing overrides or this API", + model=model, + llm_provider="", + ) + caller_body: Final = _METADATA_ADAPTER.validate_python(extra_body) if extra_body is not None else {} + return { + **data, + "extra_body": {**caller_body, "provider": dict(policy.request_body())}, + "disable_fallbacks": True, + "caching": False, + "cache": {"no-cache": True, "no-store": True}, + } reason: Final = snapshot.unavailable_models.get(offering) if reason is not None: return litellm.ServiceUnavailableError( @@ -219,17 +274,11 @@ class OfferingAccessGuard(CustomLogger): message=f"Model '{model}' is not an enabled gateway offering", model=model, llm_provider="" ) - async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: CallTypes | None) -> None: + async def async_pre_call_deployment_hook( + self, kwargs: Mapping[str, object], call_type: CallTypes | None + ) -> dict[str, object] | None: # mutable-ok: CustomLogger requires a fresh dict for request replacement model: Final = kwargs.get("model") - if not isinstance(model, str) or call_type not in ( - CallTypes.acompletion, - CallTypes.completion, - CallTypes.aresponses, - CallTypes.responses, - CallTypes.anthropic_messages, - CallTypes.aembedding, - CallTypes.embedding, - ): + if not isinstance(model, str): return metadata: Final = _METADATA_ADAPTER.validate_python( kwargs.get("litellm_metadata") or kwargs.get("metadata") or {} @@ -242,7 +291,38 @@ class OfferingAccessGuard(CustomLogger): ) deployment_id: Final = model_info.get("id") snapshot: Final = self.router.serving_snapshot() - if model not in snapshot.allowed_deployments or deployment_id not in snapshot.allowed_deployment_ids: + policy: Final = snapshot.deployment_policies.get(deployment_id) if isinstance(deployment_id, str) else None + if policy is None and call_type not in ( + CallTypes.acompletion, + CallTypes.completion, + CallTypes.aresponses, + CallTypes.responses, + CallTypes.anthropic_messages, + CallTypes.aembedding, + CallTypes.embedding, + ): + return + effective_model: Final = ( + model if model in snapshot.allowed_deployments else f"{kwargs.get('custom_llm_provider')}/{model}" + ) + if effective_model not in snapshot.allowed_deployments or deployment_id not in snapshot.allowed_deployment_ids: raise litellm.NotFoundError( message="The requested deployment is not an enabled gateway offering", model=model, llm_provider="" ) + if policy is None: + return + extra_body: Final = kwargs.get("extra_body") + if ( + call_type is None + or call_type.value not in _POLICY_CALLS + or any(kwargs.get(name) is not None for name in ("provider", "models", "route")) + or not isinstance(extra_body, Mapping) + or extra_body.get("provider") != policy.request_body() + or any(name in extra_body for name in ("model", "models", "route", "extra_body")) + ): + raise litellm.BadRequestError( + message="The request does not preserve the configured offering provider policy", + model=model, + llm_provider="", + ) + return {**kwargs, "caching": False, "cache": {"no-cache": True, "no-store": True}} diff --git a/litellm/types/proxy/model_offerings.py b/litellm/types/proxy/model_offerings.py index 87aba7b80f4..1c1794fc5e9 100644 --- a/litellm/types/proxy/model_offerings.py +++ b/litellm/types/proxy/model_offerings.py @@ -52,6 +52,34 @@ class SupplierConnection(BaseModel): return self +class OpenRouterProviderPolicy(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["openrouter"] + only: Annotated[tuple[Annotated[str, Field(min_length=1, strict=True)], ...], Field(min_length=1)] + allow_fallbacks: Literal[False] = False + + @field_validator("only") + @classmethod + def exact_endpoints(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(frozenset(value)) != len(value) or any( + endpoint != endpoint.strip() or not endpoint.strip() or "*" in endpoint or "://" in endpoint + for endpoint in value + ): + raise ValueError("Provider endpoints must be unique exact slugs, not URLs or wildcards") + return value + + @field_validator("allow_fallbacks", mode="before") + @classmethod + def no_fallbacks(cls, value: object) -> Literal[False]: + if value is not False: + raise ValueError("Offering provider policies require allow_fallbacks=false") + return False + + def request_body(self) -> Mapping[str, object]: + return MappingProxyType({"only": list(self.only), "allow_fallbacks": self.allow_fallbacks}) + + class ModelOffering(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -61,6 +89,7 @@ class ModelOffering(BaseModel): upstream_model: Annotated[str, Field(min_length=1, strict=True)] enabled: bool = Field(default=True, strict=True) model_info: OfferingModelInfo = Field(default_factory=OfferingModelInfo) + provider_policy: OpenRouterProviderPolicy | None = None @field_validator("model_name", "provider", "upstream_model") @classmethod @@ -100,4 +129,13 @@ class ModelOfferingsConfig(BaseModel): raise ValueError("Each offering must have a unique model_name") if any(offering.provider not in self.providers for offering in self.offerings): raise ValueError("Each offering must reference a configured connection") + if any( + offering.provider_policy is not None + and ( + offering.provider_policy.provider != self.providers[offering.provider].provider + or offering.model_info.mode != "chat" + ) + for offering in self.offerings + ): + raise ValueError("Provider policies require a matching supplier and explicit chat mode") return self diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index a1fdeae9835..74d9264bdb5 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -7,9 +7,17 @@ import pytest import yaml from openai import AsyncOpenAI +import litellm +from litellm.caching.caching import Cache, DualCache +from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.model_offerings import ModelOfferingsManager +from litellm.proxy.offering_router import OfferingAccessGuard +from litellm.proxy.utils import ProxyLogging from litellm.router import Router +from litellm.types.utils import CallTypesLiteral async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload(tmp_path: Path) -> None: @@ -586,3 +594,353 @@ async def test_shared_inventory_client_is_reacquired_after_cache_eviction( assert not second.client.is_closed finally: await second.close() + + +@pytest.mark.parametrize("surface", ("chat", "responses", "responses_chat", "messages")) +@pytest.mark.parametrize("stream", (False, True)) +async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, surface: str, stream: bool +) -> None: + path: Final = tmp_path / "offerings.yaml" + policy: Final = {"only": ["synthetic/fp8"], "allow_fallbacks": False} + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "supplier": { + "provider": "openrouter", + "api_base": "https://supplier.test/v1", + "api_key": "fixture-key", + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "org/backend", + "model_info": {"mode": "chat"}, + "provider_policy": {"provider": "openrouter", **policy}, + } + ], + } + ) + ) + requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/models"): + return httpx.Response(200, json={"data": []}) + requests.append(request) + body: Final = json.loads(request.content) + assert body["model"] == "org/backend" + assert body["provider"] == policy + assert body["temperature"] == 0.5 + if request.url.path.endswith("/responses"): + response: Final = { + "id": "resp_fixture", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "org/backend", + "output": [ + { + "id": "msg_fixture", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "accepted", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + if stream: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + "event: response.completed\ndata: " + + json.dumps( + { + "type": "response.completed", + "sequence_number": 1, + "response": response, + } + ) + + "\n\n" + ), + ) + return httpx.Response(200, json=response) + if stream: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + "data: " + + json.dumps( + { + "id": "fixture", + "object": "chat.completion.chunk", + "created": 1, + "model": "org/backend", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "accepted"}, + "finish_reason": "stop", + } + ], + } + ) + + "\n\ndata: [DONE]\n\n" + ), + ) + return httpx.Response( + 200, + json={ + "id": "fixture", + "object": "chat.completion", + "created": 1, + "model": "org/backend", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "accepted"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler.client = http_client + cache: Final = LLMClientCache() + cache.set_cache("async_httpx_clientssl_verify_Noneopenrouter", handler) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", cache) + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[], num_retries=0), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + guard: Final = OfferingAccessGuard(manager.router) + monkeypatch.setattr(litellm, "callbacks", [guard]) + call_type: Final[CallTypesLiteral] = ( + "acompletion" if surface == "chat" else "anthropic_messages" if surface == "messages" else "aresponses" + ) + response_cache: Final = Cache( + type="local", supported_call_types=["acompletion", "aresponses", "anthropic_messages"] + ) + monkeypatch.setattr(litellm, "cache", response_cache) + response_cache.cache.set_cache("shared-fixture-key", {"response": "unrelated endpoint output"}) + shaped: Final = await ProxyLogging(UserApiKeyCache()).pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + data={ + "model": "selected", + "stream": stream, + "extra_body": {"temperature": 0.5}, + "caching": True, + "preset_cache_key": "shared-fixture-key", + "cache": {"no-cache": False, "no-store": False}, + **({"use_chat_completions_api": True} if surface == "responses_chat" else {}), + }, + call_type=call_type, + ) + assert isinstance(shaped, dict) + result: Final = ( + await manager.router.aresponses(input="fixture", client=handler, **shaped) + if surface in ("responses", "responses_chat") + else await manager.router.anthropic_messages( + messages=[{"role": "user", "content": "fixture"}], max_tokens=10, client=handler, **shaped + ) + if surface == "messages" + else await manager.router.acompletion( + messages=[{"role": "user", "content": "fixture"}], client=handler, **shaped + ) + ) + if stream: + chunks: Final = tuple([chunk async for chunk in result]) + assert chunks + assert len(requests) == 1 + assert requests[0].url.path.endswith("/responses" if surface == "responses" else "/chat/completions") + assert response_cache.cache.get_cache("shared-fixture-key") == {"response": "unrelated endpoint output"} + + +async def test_endpoint_policy_reload_isolates_aliases_identity_and_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.utils import create_model_info_response + + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-identity-key") + path: Final = tmp_path / "offerings.yaml" + selected: Final = { + "model_name": "selected", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + "model_info": {"mode": "chat", "context_window": 4000}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + } + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openrouter", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + selected, + { + "model_name": "ordinary", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + }, + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/backend", + "context_length": 16000, + "top_provider": {"max_completion_tokens": 8000}, + } + ] + }, + ) + ) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + before: Final = manager.router.latest_snapshot() + selected_id: Final = manager.router.get_model_ids("selected") + ordinary_id: Final = manager.router.get_model_ids("ordinary") + pinned_metadata: Final = create_model_info_response("selected", "openrouter", llm_router=manager.router) + ordinary_metadata: Final = create_model_info_response("ordinary", "openrouter", llm_router=manager.router) + assert pinned_metadata["context_window"] == 4000 + assert "max_output_tokens" not in pinned_metadata + assert ordinary_metadata["context_window"] == 16000 + assert ordinary_metadata["max_output_tokens"] == 8000 + assert await manager.reload(force_inventory=True) + assert manager.router.get_model_ids("selected") == selected_id + with manager.router.pin_snapshot(): + path.write_text( + yaml.safe_dump( + { + **config, + "offerings": [ + { + **selected, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/bf16"]}, + }, + config["offerings"][1], + ], + } + ) + ) + assert await manager.reload() + assert manager.router.serving_snapshot().provider_policies == before.provider_policies + assert manager.router.get_model_ids("selected") == selected_id + assert manager.router.get_model_ids("selected") != selected_id + assert manager.router.get_model_ids("ordinary") == ordinary_id + assert manager.router.latest_snapshot().provider_policies["selected"].only == ("synthetic/bf16",) + assert "ordinary" not in manager.router.latest_snapshot().provider_policies + valid: Final = manager.router.latest_snapshot() + path.write_text( + yaml.safe_dump( + { + **config, + "providers": { + "supplier": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + } + }, + } + ) + ) + assert not await manager.reload() + assert manager.router.latest_snapshot() is valid + + +async def test_endpoint_failure_never_uses_gateway_fallback_and_tampering_never_forwards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "supplier": { + "provider": "openrouter", + "api_base": "https://supplier.test/v1", + "api_key": "fixture-key", + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "org/pinned", + "model_info": {"mode": "chat"}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + }, + { + "model_name": "ordinary", + "source": "manual", + "provider": "supplier", + "upstream_model": "org/other", + }, + ], + } + ) + ) + forwarded: Final[list[dict[str, object]]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/models"): + return httpx.Response(200, json={"data": []}) + forwarded.append(json.loads(request.content)) + return httpx.Response(503, json={"error": {"message": "synthetic endpoint unavailable", "code": 503}}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, + client=handler, + template=Router( + model_list=[], + num_retries=0, + fallbacks=[{"selected": ["ordinary"]}], + ), + ) + assert await manager.reload(initial=True) + guard: Final = OfferingAccessGuard(manager.router) + monkeypatch.setattr(litellm, "callbacks", [guard]) + shaped: Final = await guard.async_pre_call_hook( + UserAPIKeyAuth(), DualCache(), {"model": "selected"}, "acompletion" + ) + assert isinstance(shaped, dict) + with pytest.raises(litellm.ServiceUnavailableError, match="synthetic endpoint unavailable"): + await manager.router.acompletion( + messages=[{"role": "user", "content": "fixture"}], client=handler, **shaped + ) + assert tuple(body["model"] for body in forwarded) == ("org/pinned",) + for extra_body in ({"provider": {"only": ["other"]}}, {"model": "org/other"}, {"models": ["org/other"]}): + with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): + await manager.router.acompletion( + model="selected", + messages=[{"role": "user", "content": "fixture"}], + client=handler, + extra_body=extra_body, + disable_fallbacks=True, + ) + assert len(forwarded) == 1 diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 31f6f228455..25a229cf1a1 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -21,6 +21,7 @@ from litellm.proxy.offering_router import ( OfferingSnapshotMiddleware, ) from litellm.router import Router +from litellm.types.proxy.model_offerings import OpenRouterProviderPolicy from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes @@ -245,3 +246,102 @@ async def test_body_model_override_cannot_bypass_offering_authority() -> None: ) is None ) + + +@pytest.mark.parametrize( + "override", + ( + {"provider": {"only": ["other"]}}, + {"extra_body": {"provider": {"allow_fallbacks": True}}}, + {"extra_body": {"extra_body": {"provider": {"only": ["other"]}}}}, + {"models": ["other"]}, + {"route": "fallback"}, + {"extra_body": {"models": ["other"]}}, + {"extra_body": {"route": "fallback"}}, + {"fallbacks": [{"selected": ["other"]}]}, + {"context_window_fallbacks": [{"selected": ["other"]}]}, + {"content_policy_fallbacks": [{"selected": ["other"]}]}, + {"specific_deployment": True}, + {"model_id": "unrestricted"}, + {"disable_fallbacks": False}, + {"mock_response": "bypassed"}, + ), +) +async def test_pinned_policy_rejects_caller_routing_even_through_alias(override: dict[str, object]) -> None: + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + view: Final = OfferingRouterView( + OfferingServingSnapshot( + Router(model_list=[]), + frozenset({"selected", "ordinary"}), + MappingProxyType({}), + frozenset(), + provider_policies=MappingProxyType({"selected": policy}), + ) + ) + guard: Final = OfferingAccessGuard(view) + auth: Final = UserAPIKeyAuth(aliases={"alias": "selected"}) + for call_type in ("acompletion", "aresponses", "anthropic_messages"): + rejected: Final = await guard.async_pre_call_hook(auth, DualCache(), {"model": "alias", **override}, call_type) + assert isinstance(rejected, litellm.BadRequestError) + assert "caller" in rejected.message + + +async def test_pinned_policy_shapes_requests_without_mutation_and_blocks_unsupported_apis() -> None: + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + view: Final = OfferingRouterView( + OfferingServingSnapshot( + Router(model_list=[]), + frozenset({"selected", "ordinary"}), + MappingProxyType({}), + frozenset(), + provider_policies=MappingProxyType({"selected": policy}), + ) + ) + guard: Final = OfferingAccessGuard(view) + original: Final = {"model": "selected", "extra_body": {"temperature": 0.5}} + assert await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), original, "acompletion") == { + "model": "selected", + "extra_body": {"temperature": 0.5, "provider": {"only": ["synthetic/fp8"], "allow_fallbacks": False}}, + "disable_fallbacks": True, + "caching": False, + "cache": {"no-cache": True, "no-store": True}, + } + assert original == {"model": "selected", "extra_body": {"temperature": 0.5}} + assert ( + await guard.async_pre_call_hook( + UserAPIKeyAuth(), DualCache(), {"model": "ordinary", "provider": {"only": ["other"]}}, "acompletion" + ) + is None + ) + assert isinstance( + await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": "selected"}, "aembedding"), + litellm.BadRequestError, + ) + assert isinstance( + await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": "openrouter/backend"}, "acompletion"), + litellm.NotFoundError, + ) + + +@pytest.mark.parametrize("call_type", (None, CallTypes.aresponses_websocket, CallTypes.aimage_generation)) +async def test_pinned_deployment_rejects_apis_that_cannot_carry_policy(call_type: CallTypes | None) -> None: + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + view: Final = OfferingRouterView( + OfferingServingSnapshot( + Router(model_list=[]), + frozenset({"selected"}), + MappingProxyType({}), + frozenset({"openrouter/backend"}), + frozenset({"selected-id"}), + deployment_policies=MappingProxyType({"selected-id": policy}), + ) + ) + with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): + await OfferingAccessGuard(view).async_pre_call_deployment_hook( + { + "model": "openrouter/backend", + "metadata": {"model_info": {"id": "selected-id"}}, + "extra_body": {"provider": dict(policy.request_body())}, + }, + call_type, + ) diff --git a/tests/unit/types/proxy/test_model_offerings.py b/tests/unit/types/proxy/test_model_offerings.py index b0999826c1a..019dc5fa093 100644 --- a/tests/unit/types/proxy/test_model_offerings.py +++ b/tests/unit/types/proxy/test_model_offerings.py @@ -3,7 +3,7 @@ from typing import Final import pytest from pydantic import ValidationError -from litellm.types.proxy.model_offerings import ModelOfferingsConfig +from litellm.types.proxy.model_offerings import ModelOfferingsConfig, OpenRouterProviderPolicy @pytest.mark.parametrize( @@ -48,3 +48,45 @@ def test_invalid_operator_edits_are_rejected(change) -> None: } with pytest.raises(ValidationError): ModelOfferingsConfig.model_validate({**config, **change}) + + +@pytest.mark.parametrize( + "change", + ( + {"provider": "openai"}, + {"only": []}, + {"only": ["endpoint", "endpoint"]}, + {"only": ["endpoint "]}, + {"only": ["https://supplier.test"]}, + {"only": ["supplier/*"]}, + {"allow_fallbacks": True}, + {"allow_fallbacks": 0}, + {"api_key": "not-allowed"}, + ), +) +def test_provider_policy_rejects_nonexact_or_unsafe_configuration(change: dict[str, object]) -> None: + with pytest.raises(ValidationError): + OpenRouterProviderPolicy.model_validate( + {"provider": "openrouter", "only": ["synthetic/fp8"], "allow_fallbacks": False, **change} + ) + + +@pytest.mark.parametrize("supplier,mode", (("openai", "chat"), ("openrouter", None), ("openrouter", "embedding"))) +def test_provider_policy_requires_supported_supplier_and_api(supplier: str, mode: str | None) -> None: + with pytest.raises(ValidationError, match="matching supplier and explicit chat mode"): + ModelOfferingsConfig.model_validate( + { + "version": 1, + "providers": {"supplier": {"provider": supplier, "api_base": "https://supplier.test/v1"}}, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "backend", + "model_info": {"mode": mode}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + } + ], + } + ) From 174317452d716ef5fd63ae5e0b886f7c6e4d5b5b Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 12 Oct 2026 14:55:13 +0200 Subject: [PATCH 20/34] fix(proxy): reject endpoint policy parameter-drop overrides --- examples/model_offerings/README.md | 2 +- litellm/proxy/offering_router.py | 42 ++++++++++++++---------- tests/unit/proxy/test_model_offerings.py | 24 ++++++++++++++ tests/unit/proxy/test_offering_router.py | 7 ++++ 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index b05e3e9b9cd..6d1f856af2d 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -91,7 +91,7 @@ offerings: Append this entry to the external file's existing `offerings`, using its configured OpenRouter connection name. The example limits are synthetic. Verify endpoint-specific values before enabling it. `only` contains provider slugs, not URLs, and a single full slug pins one endpoint. Multiple explicit slugs intentionally allow any endpoint in that list; `allow_fallbacks` must remain false -The gateway merges its policy into harmless caller `extra_body` fields and verifies it again at deployment dispatch. Caller `provider`, `models`, `route`, fallback controls, nested body overrides and mock responses are rejected for policy-controlled offerings, even through aliases. Caller deployment selectors are rejected throughout external mode. Policy-controlled requests disable LiteLLM model-group fallbacks as well as OpenRouter provider fallback. Unavailable endpoints fail instead of serving a different offering. Retries may still contact the same pinned endpoint +The gateway merges its policy into harmless caller `extra_body` fields and verifies it again at deployment dispatch. Caller `provider`, `models`, `route`, fallback controls, parameter-dropping/allowlist controls, nested body overrides and mock responses are rejected for policy-controlled offerings, even through aliases. Caller deployment selectors are rejected throughout external mode. Policy-controlled requests disable LiteLLM model-group fallbacks as well as OpenRouter provider fallback. Unavailable endpoints fail instead of serving a different offering. Retries may still contact the same pinned endpoint Policy-controlled requests disable response-cache reads and writes, overriding caller cache settings. Existing response-cache keys do not reliably separate provider endpoints; cached output from another endpoint must not satisfy a pinned request diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index ed769238ee0..80f2c01157b 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -38,25 +38,29 @@ _MODEL_MUTATIONS: Final = frozenset(("set_model_list", "add_deployment", "upsert _VIEW_MEMBERS: Final = frozenset(("publish_snapshot", "pin_snapshot", "serving_snapshot", "latest_snapshot")) _METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _POLICY_CALLS: Final = frozenset(("acompletion", "completion", "aresponses", "responses", "anthropic_messages")) -_ROUTING_OVERRIDES: Final = frozenset( - ( - "provider", - "models", - "route", - "fallbacks", - "default_fallbacks", - "context_window_fallbacks", - "content_policy_fallbacks", - "model_list", - "deployment_id", - "specific_deployment", - "model_id", - "disable_fallbacks", - "mock_response", - "mock_tool_calls", - "mock_timeout", - "mock_provider_error", +_PARAMETER_CONTROLS: Final = frozenset(("additional_drop_params", "drop_params", "allowed_openai_params")) +_ROUTING_OVERRIDES: Final = ( + frozenset( + ( + "provider", + "models", + "route", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "model_list", + "deployment_id", + "specific_deployment", + "model_id", + "disable_fallbacks", + "mock_response", + "mock_tool_calls", + "mock_timeout", + "mock_provider_error", + ) ) + | _PARAMETER_CONTROLS ) @@ -315,9 +319,11 @@ class OfferingAccessGuard(CustomLogger): if ( call_type is None or call_type.value not in _POLICY_CALLS + or any(kwargs.get(name) is not None for name in _PARAMETER_CONTROLS) or any(kwargs.get(name) is not None for name in ("provider", "models", "route")) or not isinstance(extra_body, Mapping) or extra_body.get("provider") != policy.request_body() + or bool(_PARAMETER_CONTROLS.intersection(extra_body)) or any(name in extra_body for name in ("model", "models", "route", "extra_body")) ): raise litellm.BadRequestError( diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 74d9264bdb5..8aa5bd4aa75 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -762,6 +762,30 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( assert len(requests) == 1 assert requests[0].url.path.endswith("/responses" if surface == "responses" else "/chat/completions") assert response_cache.cache.get_cache("shared-fixture-key") == {"response": "unrelated endpoint output"} + for control in ( + {"additional_drop_params": ["provider"]}, + {"additional_drop_params": ["extra_body"]}, + {"additional_drop_params": ["extra_body.provider"]}, + {"additional_drop_params": ["provider.only"]}, + {"drop_params": True}, + {"allowed_openai_params": ["provider"]}, + {"extra_body": {"provider": policy, "temperature": 0.5, "additional_drop_params": ["provider"]}}, + ): + with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): + if surface in ("responses", "responses_chat"): + await manager.router.aresponses(input="fixture", client=handler, **{**shaped, **control}) + elif surface == "messages": + await manager.router.anthropic_messages( + messages=[{"role": "user", "content": "fixture"}], + max_tokens=10, + client=handler, + **{**shaped, **control}, + ) + else: + await manager.router.acompletion( + messages=[{"role": "user", "content": "fixture"}], client=handler, **{**shaped, **control} + ) + assert len(requests) == 1 async def test_endpoint_policy_reload_isolates_aliases_identity_and_metadata( diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 25a229cf1a1..263c8d3c08e 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -265,6 +265,13 @@ async def test_body_model_override_cannot_bypass_offering_authority() -> None: {"model_id": "unrestricted"}, {"disable_fallbacks": False}, {"mock_response": "bypassed"}, + {"additional_drop_params": ["provider"]}, + {"additional_drop_params": ["extra_body"]}, + {"additional_drop_params": ["extra_body.provider"]}, + {"additional_drop_params": ["provider.only"]}, + {"additional_drop_params": ["provider.allow_fallbacks"]}, + {"drop_params": True}, + {"allowed_openai_params": ["provider"]}, ), ) async def test_pinned_policy_rejects_caller_routing_even_through_alias(override: dict[str, object]) -> None: From 10b8847e48f7e602ee0f9bfb323c677cf23cffcc Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 12 Oct 2026 15:17:27 +0200 Subject: [PATCH 21/34] fix(proxy): tighten policy-drop regression assertions --- litellm/proxy/offering_router.py | 2 +- tests/unit/proxy/test_model_offerings.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index 80f2c01157b..971524866d4 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -323,7 +323,7 @@ class OfferingAccessGuard(CustomLogger): or any(kwargs.get(name) is not None for name in ("provider", "models", "route")) or not isinstance(extra_body, Mapping) or extra_body.get("provider") != policy.request_body() - or bool(_PARAMETER_CONTROLS.intersection(extra_body)) + or any(name in extra_body for name in _PARAMETER_CONTROLS) or any(name in extra_body for name in ("model", "models", "route", "extra_body")) ): raise litellm.BadRequestError( diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 8aa5bd4aa75..350efa80fd5 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -772,19 +772,20 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( {"extra_body": {"provider": policy, "temperature": 0.5, "additional_drop_params": ["provider"]}}, ): with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): - if surface in ("responses", "responses_chat"): - await manager.router.aresponses(input="fixture", client=handler, **{**shaped, **control}) - elif surface == "messages": - await manager.router.anthropic_messages( + await ( + manager.router.aresponses(input="fixture", client=handler, **{**shaped, **control}) + if surface in ("responses", "responses_chat") + else manager.router.anthropic_messages( messages=[{"role": "user", "content": "fixture"}], max_tokens=10, client=handler, **{**shaped, **control}, ) - else: - await manager.router.acompletion( + if surface == "messages" + else manager.router.acompletion( messages=[{"role": "user", "content": "fixture"}], client=handler, **{**shaped, **control} ) + ) assert len(requests) == 1 From 65f39ea1ab4a210b15581aa8da8dfbc2c05bc4cc Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Tue, 13 Oct 2026 14:12:21 +0200 Subject: [PATCH 22/34] fix(proxy): publish selected models for wildcard permissions --- .github/workflows/test-rust.yml | 2 - litellm/proxy/auth/model_checks.py | 18 ++- tests/unit/proxy/auth/test_model_checks.py | 32 +++++ tests/unit/proxy/test_model_offerings.py | 156 ++++++++++++++++++++- 4 files changed, 200 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 39d9ce2a80e..f8a7c38ef79 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -99,8 +99,6 @@ jobs: # Trim debuginfo to keep the job's target dir within the runner's disk env: CARGO_PROFILE_DEV_DEBUG: line-tables-only - CARGO_PROFILE_TEST_DEBUG: "0" - CARGO_INCREMENTAL: "0" defaults: run: working-directory: litellm-rust diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 3101f19ced2..81f9d0108cf 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -8,6 +8,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import is_model_allowed_by_pattern from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group @@ -206,12 +207,12 @@ def get_complete_model_list( - If key list is empty -> defer to team list - If team list is empty -> defer to proxy model list - If list contains wildcard -> return known provider models + If list contains wildcard -> include matching proxy names and known provider models """ - unique_models: Final = [] + unique_models: Final[list[str]] = [] - def append_unique(models): + def append_unique(models: Sequence[str]) -> None: for model in models: if model not in unique_models and model != SpecialModelNames.no_default_models.value: unique_models.append(model) @@ -239,6 +240,15 @@ def get_complete_model_list( model_access_groups_to_return.append(model) return model_access_groups_to_return + granted_proxy_models: Final = tuple( + dict.fromkeys( + model + for model in proxy_model_list + if not _check_wildcard_routing(model) + and any(is_model_allowed_by_pattern(model, grant) for grant in unique_models) + ) + ) + all_wildcard_models: Final = _get_wildcard_models( unique_models=unique_models, return_wildcard_routes=return_wildcard_routes, @@ -248,7 +258,7 @@ def get_complete_model_list( complete_model_list: Final = unique_models + all_wildcard_models - return complete_model_list + return complete_model_list + [model for model in granted_proxy_models if model not in complete_model_list] def _hydrate_litellm_credential_name( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 9f9a6687210..e401d563089 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -6,6 +6,38 @@ import pytest import respx +@pytest.mark.parametrize( + "key_grants,team_grants,groups_only,expected", + ( + (("public/*",), (), False, ("public/selected",)), + (("public/sel*",), (), False, ("public/selected",)), + (("*",), (), False, ("public/selected", "private/selected")), + ((), ("public/*",), False, ("public/selected",)), + (("private/selected",), ("public/*",), False, ("private/selected",)), + (("denied/*",), (), False, ()), + (("public/*",), (), True, ()), + (("no-default-models",), ("public/*",), False, ()), + ), +) +def test_wildcard_grants_list_deployed_public_names_without_provider_inventory( + key_grants: tuple[str, ...], + team_grants: tuple[str, ...], + groups_only: bool, + expected: tuple[str, ...], +) -> None: + from litellm.proxy.auth.model_checks import get_complete_model_list + + result: Final = get_complete_model_list( + key_models=key_grants, + team_models=team_grants, + proxy_model_list=["public/selected", "public/selected", "private/selected", "public/*"], + user_model=None, + infer_model_from_keys=False, + only_model_access_groups=groups_only, + ) + assert tuple(result) == expected + + def test_get_team_models_for_all_models_and_team_only_models(): from litellm.proxy.auth.model_checks import get_team_models diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 350efa80fd5..a751d27f320 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -1,25 +1,177 @@ import json from pathlib import Path +from types import MappingProxyType from typing import Final import httpx import pytest import yaml +from fastapi import FastAPI from openai import AsyncOpenAI +from starlette.requests import Request import litellm from litellm.caching.caching import Cache, DualCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.model_offerings import ModelOfferingsManager -from litellm.proxy.offering_router import OfferingAccessGuard +from litellm.proxy.offering_router import OfferingAccessGuard, OfferingSnapshotMiddleware from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.utils import CallTypesLiteral +@pytest.mark.parametrize("catalog_state", ("missing", "known", "different_route_provider")) +async def test_wildcard_listing_tracks_selected_offerings_and_pinned_reload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, catalog_state: str +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + path: Final = tmp_path / "offerings.yaml" + selected: Final = { + "model_name": "openrouter/org/new-model", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/new-model", + "model_info": {"mode": "chat", "context_window": 4000}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + } + existing: Final = { + "model_name": "chatgpt/existing", + "source": "manual", + "provider": "secondary", + "upstream_model": "org/existing", + } + other_existing: Final = tuple( + {**existing, "model_name": name} for name in ("chatgpt/second", "chatgpt/third", "chatgpt/fourth") + ) + existing_names: Final = frozenset((existing["model_name"], *(row["model_name"] for row in other_existing))) + denied: Final = {**existing, "model_name": "restricted/other"} + disabled: Final = {**selected, "model_name": "openrouter/org/disabled", "enabled": False} + absent: Final = {**selected, "model_name": "openrouter/org/absent", "upstream_model": "org/absent"} + config: Final = { + "version": 1, + "providers": { + "supplier": { + "provider": "openrouter", + "api_base": "https://supplier.test/v1", + "api_key": "private-fixture-key", + }, + "secondary": {"provider": "openai", "api_base": "https://secondary.test/v1"}, + }, + "offerings": [existing, *other_existing, denied, disabled, absent], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response(200, json={"data": [{"id": "org/new-model"}, {"id": "org/unselected"}]}) + ) + ) as client: + handler.client = client + template: Final = Router( + model_list=[], + model_group_alias={"openrouter/*": existing["model_name"]} + if catalog_state == "different_route_provider" + else {}, + ) + manager: Final = ModelOfferingsManager(path=path, template=template, client=handler, clock=lambda: 0.0) + assert await manager.reload(initial=True) + monkeypatch.setattr(proxy_server, "llm_router", manager.router) + monkeypatch.setattr(proxy_server, "llm_model_list", manager.router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(litellm, "callbacks", [OfferingAccessGuard(manager.router)]) + monkeypatch.setattr( + litellm, + "models_by_provider", + { + "openrouter": ["org/catalog-only"] if catalog_state == "missing" else [selected["model_name"]], + "chatgpt": sorted(existing_names), + "openai": ["fixture/catalog-only"], + }, + ) + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + assert (selected["model_name"] in get_known_models_from_wildcard("openrouter/*")) == ( + catalog_state != "missing" + ) + wildcard: Final = UserAPIKeyAuth( + api_key="wildcard", user_role=LitellmUserRoles.INTERNAL_USER, models=["openrouter/*", "chatgpt/*"] + ) + exact: Final = UserAPIKeyAuth( + api_key="exact", user_role=LitellmUserRoles.INTERNAL_USER, models=sorted(existing_names) + ) + denied_key: Final = UserAPIKeyAuth( + api_key="denied", user_role=LitellmUserRoles.INTERNAL_USER, models=["denied/*"] + ) + team_key: Final = UserAPIKeyAuth( + api_key="team", user_role=LitellmUserRoles.INTERNAL_USER, team_id="team-one", team_models=["openrouter/*"] + ) + alias_key: Final = UserAPIKeyAuth( + api_key="alias", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["shortcut"], + aliases={"shortcut": selected["model_name"]}, + ) + admin: Final = UserAPIKeyAuth(api_key="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + credentials: Final = MappingProxyType( + {auth.api_key: auth for auth in (wildcard, exact, denied_key, team_key, alias_key, admin)} + ) + application: Final = FastAPI() + application.include_router(proxy_server.router) + application.add_middleware(OfferingSnapshotMiddleware, router_getter=lambda: manager.router) + + async def scoped_auth(request: Request) -> UserAPIKeyAuth: + if edit := request.headers.get("x-fixture-offerings"): + path.write_text(yaml.safe_dump(json.loads(edit))) + assert await manager.reload() + return credentials[request.headers["authorization"].removeprefix("Bearer ")] + + application.dependency_overrides[proxy_server.user_api_key_auth] = scoped_auth + + async def listed(auth: UserAPIKeyAuth, edit: str | None = None) -> frozenset[str]: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=application), base_url="http://gateway.test" + ) as gateway: + response: Final = await gateway.get( + "/v1/models", + headers={ + "Authorization": f"Bearer {auth.api_key}", + **({"x-fixture-offerings": edit} if edit is not None else {}), + }, + ) + assert response.status_code == 200, response.text + assert "private-fixture-key" not in response.text + assert "supplier.test" not in response.text + assert "secondary.test" not in response.text + assert "org/existing" not in response.text + assert "synthetic/fp8" not in response.text + assert "offering-" not in response.text + return frozenset(row["id"] for row in response.json()["data"]) + + assert await listed(exact) == existing_names + assert await listed(admin) == existing_names | {"restricted/other"} + assert await listed(admin, json.dumps({**config, "offerings": [*config["offerings"], selected]})) == ( + existing_names | {"restricted/other"} + ) + assert await listed(admin) == existing_names | {"restricted/other", selected["model_name"]} + assert await listed(wildcard) == existing_names | {selected["model_name"]} + assert await listed(exact) == existing_names + assert await listed(denied_key) == set() + assert await listed(team_key) == {selected["model_name"]} + assert await listed(alias_key) == {"shortcut"} + assert await listed(wildcard, json.dumps(config)) == existing_names | {selected["model_name"]} + assert await listed(wildcard) == existing_names + assert await listed(alias_key) == set() + assert await listed(team_key) == set() + assert await listed(admin) == existing_names | {"restricted/other"} + + async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload(tmp_path: Path) -> None: path: Final = tmp_path / "offerings.yaml" config: Final = { From 6e7274dd18bbdc1b28133768fbdefb0cf0d836b9 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Tue, 13 Oct 2026 18:07:32 +0200 Subject: [PATCH 23/34] refactor(proxy): consolidate offering metadata and policies into native models --- litellm/litellm_core_utils/core_helpers.py | 5 +- litellm/llms/chatgpt/model_info.py | 32 ++- litellm/llms/model_inventory.py | 9 +- litellm/llms/openai_like/model_info.py | 23 +-- .../llms/openrouter/chat/transformation.py | 6 + .../openrouter/responses/transformation.py | 9 +- .../vercel_ai_gateway/chat/transformation.py | 6 +- litellm/main.py | 6 +- litellm/proxy/_types.py | 5 +- .../common_utils/openai_endpoint_utils.py | 2 +- litellm/proxy/model_discovery.py | 8 +- litellm/proxy/model_offerings.py | 26 +-- litellm/proxy/offering_router.py | 17 +- litellm/proxy/proxy_server.py | 10 +- litellm/proxy/utils.py | 71 +++---- litellm/router.py | 10 +- .../router_utils/model_request_defaults.py | 4 +- litellm/types/llms/openrouter.py | 35 ++++ litellm/types/proxy/model_listing.py | 7 +- litellm/types/proxy/model_metadata.py | 135 +------------ litellm/types/proxy/model_offerings.py | 33 +-- litellm/types/router.py | 188 +++++++++++++++++- litellm/types/utils.py | 6 + litellm/utils.py | 1 + tests/unit/llms/test_model_inventory.py | 63 +++++- tests/unit/proxy/test_model_offerings.py | 75 ++++++- tests/unit/proxy/test_offering_router.py | 57 +++--- tests/unit/test_model_prices_schema.py | 29 +++ tests/unit/types/test_router.py | 61 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 110 +++++++++- 30 files changed, 698 insertions(+), 351 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e857461785c..888ee644a39 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -166,7 +166,7 @@ def is_expected_client_error(exception: BaseException | None) -> bool: return 400 <= status < 500 -def coerce_token_limit(value: object) -> int | None: +def coerce_token_limit(value: object, *, strict_positive: bool = False) -> int | None: """ Coerce a max_input_tokens / max_output_tokens value to an int, treating a malformed value as absent. @@ -178,11 +178,14 @@ def coerce_token_limit(value: object) -> int | None: Args: value: The raw configured or cost-map value + strict_positive: Require positive integers without legacy numeric coercion Returns: The value as an int, or None if it is missing or not a usable number. Bools are rejected because True/False is never a meaningful token limit. """ + if strict_positive: + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None if isinstance(value, bool): return None if isinstance(value, int): diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index 2a3707de87e..c88d8f96db0 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -9,6 +9,7 @@ import httpx from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.core_helpers import coerce_token_limit from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.types.proxy.model_inventory import ( @@ -17,7 +18,7 @@ from litellm.types.proxy.model_inventory import ( SupplierInventoryUnavailable, SupplierModelInventory, ) -from litellm.types.proxy.model_metadata import GatewayModelMetadata +from litellm.types.router import ModelMetadata from .authenticator import Authenticator from .common_utils import get_chatgpt_default_headers @@ -32,7 +33,7 @@ class _ReasoningLevel(BaseModel): effort: str -class _ChatGPTModel(GatewayModelMetadata): +class _ChatGPTModel(ModelMetadata): slug: str max_context_window: int | None = None default_reasoning_level: str | None = None @@ -44,28 +45,21 @@ class _ChatGPTModel(GatewayModelMetadata): @field_validator("*", mode="before") @classmethod def strict_supplier_metadata(cls, value: object, info: ValidationInfo) -> object: - normalized: Final = ( - None - if info.field_name in ("context_window", "max_input_tokens", "max_output_tokens", "max_context_window") - and type(value) is int - and value == 0 - else value + field_name: Final = ( + "context_window" + if info.field_name == "max_context_window" + else "supports_parallel_function_calling" + if info.field_name == "supports_parallel_tool_calls" + else info.field_name ) - if info.field_name in GatewayModelMetadata.model_fields or info.field_name == "max_context_window": - cls.validate_field_input( - "context_window" if info.field_name == "max_context_window" else info.field_name, normalized - ) - if info.field_name == "supports_parallel_tool_calls": - cls.validate_field_input("supports_parallel_function_calling", normalized) - return normalized + return ModelMetadata.validate_supplier_field(field_name, value) @field_validator("max_context_window", mode="before") @classmethod def maximum_context(cls, value: object) -> int | None: - return GatewayModelMetadata.model_validate({"context_window": value}).context_window + return coerce_token_limit(value, strict_positive=True) def metadata(self) -> Mapping[str, object]: - maximum: Final = GatewayModelMetadata(context_window=self.max_context_window).context_window native_efforts: Final = ( self.reasoning_effort_levels if self.reasoning_effort_levels is not None @@ -93,7 +87,7 @@ class _ChatGPTModel(GatewayModelMetadata): if native_default == "persistent" else native_default ) - normalized: Final = GatewayModelMetadata( + normalized: Final = ModelMetadata( max_input_tokens=self.max_input_tokens, max_output_tokens=self.max_output_tokens, supports_function_calling=self.supports_function_calling, @@ -119,7 +113,7 @@ class _ChatGPTModel(GatewayModelMetadata): ) native_windows: Final = { "codex_context_window": self.context_window, - "codex_max_context_window": maximum, + "codex_max_context_window": self.max_context_window, } return MappingProxyType( { diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py index ebef987a3bc..89551a794a1 100644 --- a/litellm/llms/model_inventory.py +++ b/litellm/llms/model_inventory.py @@ -33,10 +33,11 @@ def prevent_supplier_device_login() -> Generator[None]: def model_inventory_api_base(provider: str, api_base: str | None) -> str | None: if provider == "chatgpt": return chatgpt_model_inventory_api_base(api_base) - return api_base or { - "openrouter": "https://openrouter.ai/api/v1", - "vercel_ai_gateway": "https://ai-gateway.vercel.sh/v1", - }.get(provider) + if provider == "openrouter": + return litellm.OpenrouterConfig.get_api_base(api_base) + if provider == "vercel_ai_gateway": + return litellm.VercelAIGatewayConfig.get_api_base(api_base) + return api_base def model_inventory_identity(provider: str) -> str | None: diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 1633d8c3c32..724775b671a 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -8,6 +8,7 @@ import httpx from pydantic import BeforeValidator, ConfigDict, ValidationError, ValidationInfo, field_validator from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.core_helpers import coerce_token_limit from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.model_inventory import ( @@ -16,7 +17,7 @@ from litellm.types.proxy.model_inventory import ( SupplierInventoryUnavailable, SupplierModelInventory, ) -from litellm.types.proxy.model_metadata import GatewayModelMetadata +from litellm.types.router import ModelMetadata from litellm.utils import add_path_to_api_base MODEL_INFO_REFRESH_SECONDS: Final = 300 @@ -30,8 +31,9 @@ _EMPTY_LIMITS: Final[Mapping[str, object]] = MappingProxyType({}) def _positive_limit(value: object) -> int | None: if value is None or type(value) is int and value == 0: return None - if type(value) is int and value > 0: - return value + limit: Final = coerce_token_limit(value, strict_positive=True) + if type(value) is int and limit is not None: + return limit raise ValueError("Supplier token limits must be nonnegative integers or null") @@ -73,7 +75,7 @@ class _ReasoningOption(LiteLLMBaseModel): values: Sequence[str] | None = None -class _ModelCard(GatewayModelMetadata): +class _ModelCard(ModelMetadata): id: str type: str | None = None mode: str | None = None @@ -90,16 +92,7 @@ class _ModelCard(GatewayModelMetadata): @field_validator("*", mode="before") @classmethod def strict_supplier_metadata(cls, value: object, info: ValidationInfo) -> object: - normalized: Final = ( - None - if info.field_name in ("context_window", "max_input_tokens", "max_output_tokens") - and type(value) is int - and value == 0 - else value - ) - if info.field_name in GatewayModelMetadata.model_fields: - cls.validate_field_input(info.field_name, normalized) - return normalized + return ModelMetadata.validate_supplier_field(info.field_name, value) def token_limits(self, provider: str) -> Mapping[str, object]: context: Final = self.context_window or self.max_model_len or self.context_length @@ -108,7 +101,7 @@ class _ModelCard(GatewayModelMetadata): supported_reasoning: Final = ( "reasoning" in parameters or "reasoning_effort" in parameters if parameters is not None else None ) - metadata: Final = GatewayModelMetadata( + metadata: Final = ModelMetadata( context_window=( self.top_provider.context_length or context if provider == "openrouter" and self.top_provider is not None diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index c63863bbca4..34bab2f8c73 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -12,9 +12,11 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx +import litellm from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.llms.openrouter import OpenRouterErrorMessage from litellm.types.utils import ModelResponse, ModelResponseStream @@ -38,6 +40,10 @@ class CacheControlSupportedModels(str, Enum): class OpenrouterConfig(OpenAIGPTConfig): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + def get_supported_openai_params(self, model: str) -> list: """ Include reasoning parameters regardless of model-map capabilities. diff --git a/litellm/llms/openrouter/responses/transformation.py b/litellm/llms/openrouter/responses/transformation.py index aeb5b09e14a..e21e0ee0ea5 100644 --- a/litellm/llms/openrouter/responses/transformation.py +++ b/litellm/llms/openrouter/responses/transformation.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openrouter.chat.transformation import OpenrouterConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( ResponseCompletedEvent, @@ -134,13 +135,7 @@ class OpenRouterResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str | None, litellm_params: dict, ) -> str: - api_base = ( - api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - ) - - api_base = api_base.rstrip("/") - - return f"{api_base}/responses" + return f"{OpenrouterConfig.get_api_base(api_base).rstrip('/')}/responses" def supports_native_websocket(self) -> bool: """OpenRouter does not support native WebSocket for Responses API""" diff --git a/litellm/llms/vercel_ai_gateway/chat/transformation.py b/litellm/llms/vercel_ai_gateway/chat/transformation.py index 19c4c2659ca..1c42e1b3905 100644 --- a/litellm/llms/vercel_ai_gateway/chat/transformation.py +++ b/litellm/llms/vercel_ai_gateway/chat/transformation.py @@ -20,6 +20,10 @@ from ..common_utils import VercelAIGatewayException class VercelAIGatewayConfig(OpenAIGPTConfig): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return api_base or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") or "https://ai-gateway.vercel.sh/v1" + @property def custom_llm_provider(self) -> str | None: return "vercel_ai_gateway" @@ -33,7 +37,7 @@ class VercelAIGatewayConfig(OpenAIGPTConfig): def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: - api_base = api_base or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") or "https://ai-gateway.vercel.sh/v1" + api_base = self.get_api_base(api_base) user_api_key = api_key or get_secret_str("VERCEL_AI_GATEWAY_API_KEY") or get_secret_str("VERCEL_OIDC_TOKEN") return api_base, user_api_key diff --git a/litellm/main.py b/litellm/main.py index 474d82e8ef1..8fe4e188e42 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3386,7 +3386,7 @@ def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchR stream: Final = ctx.stream timeout: Final = ctx.timeout - api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + api_base = litellm.OpenrouterConfig.get_api_base(api_base) api_key = ( api_key @@ -6698,9 +6698,7 @@ def embedding( headers=headers, ) elif custom_llm_provider == "openrouter": - api_base = ( - api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - ) + api_base = litellm.OpenrouterConfig.get_api_base(api_base) api_key = ( api_key diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a7087e3e730..1374fc5d0aa 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -85,9 +85,8 @@ from litellm.types.proxy.carried_budget_state import ( UserBudgetSnapshot as UserBudgetSnapshot, # noqa: PLC0414 # public re-export ) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry -from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings -from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterConfig +from litellm.types.router import AllowedModelRegion, ModelMetadata, RouterErrors, UpdateRouterConfig from litellm.types.router_weights import validate_router_settings_dict from litellm.types.secret_managers.main import KeyManagementSystem from litellm.types.utils import ( @@ -1105,7 +1104,7 @@ class ModelInfoDelete(LiteLLMPydanticObjectBase): class ModelInfo( # frozen-ok: native model management assigns team_public_model_name after validation - GatewayModelMetadata, LiteLLMPydanticObjectBase + ModelMetadata, LiteLLMPydanticObjectBase ): id: str | None mode: Literal["embedding", "chat", "completion"] | None diff --git a/litellm/proxy/common_utils/openai_endpoint_utils.py b/litellm/proxy/common_utils/openai_endpoint_utils.py index 320093cfbb4..3c3d5d105e7 100644 --- a/litellm/proxy/common_utils/openai_endpoint_utils.py +++ b/litellm/proxy/common_utils/openai_endpoint_utils.py @@ -52,7 +52,7 @@ def remove_sensitive_info_from_deployment( _excluded: Final = (excluded_keys or set()) | _rate_limit_config_keys return { - **deployment_dict, + **{key: value for key, value in deployment_dict.items() if key != "provider_policy"}, "litellm_params": SENSITIVE_DATA_MASKER.mask_dict( {key: value for key, value in deployment_dict["litellm_params"].items() if key not in _private_params}, excluded_keys=_excluded, diff --git a/litellm/proxy/model_discovery.py b/litellm/proxy/model_discovery.py index 5fa0a71d6a3..2406e3f5175 100644 --- a/litellm/proxy/model_discovery.py +++ b/litellm/proxy/model_discovery.py @@ -4,13 +4,13 @@ from typing import Final, Literal from pydantic import TypeAdapter from litellm.types.proxy.model_listing import ModelDiscoveryInfo -from litellm.types.proxy.model_metadata import GatewayModelMetadata +from litellm.types.router import ModelMetadata _ADAPTER: Final = TypeAdapter(ModelDiscoveryInfo) def guaranteed_token_limit( - sources: tuple[GatewayModelMetadata, ...], field: Literal["context_window", "max_input_tokens", "max_output_tokens"] + sources: tuple[ModelMetadata, ...], field: Literal["context_window", "max_input_tokens", "max_output_tokens"] ) -> int | None: values: Final = tuple( source.context_window @@ -38,7 +38,7 @@ def _common_list(values: tuple[Sequence[str] | None, ...]) -> Sequence[str] | No return tuple(item for item in first if all(value is not None and item in value for value in values)) -def discover_model_metadata(sources: tuple[GatewayModelMetadata, ...]) -> ModelDiscoveryInfo: +def discover_model_metadata(sources: tuple[ModelMetadata, ...]) -> ModelDiscoveryInfo: efforts: Final = _common_list(tuple(source.reasoning_effort_levels for source in sources)) defaults: Final = tuple(source.default_reasoning_effort for source in sources) default: Final = defaults[0] if defaults and all(value == defaults[0] for value in defaults) else None @@ -49,7 +49,7 @@ def discover_model_metadata(sources: tuple[GatewayModelMetadata, ...]) -> ModelD if request_defaults and all(value == request_defaults[0] for value in request_defaults) else None ) - metadata: Final = GatewayModelMetadata( + metadata: Final = ModelMetadata( context_window=guaranteed_token_limit(sources, "context_window"), supports_function_calling=_common_boolean(tuple(source.supports_function_calling for source in sources)), supports_parallel_function_calling=_common_boolean( diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index 4b17e0773d1..25c79c3067d 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -28,8 +28,8 @@ from litellm.router import Router from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory -from litellm.types.proxy.model_metadata import GatewayModelMetadata from litellm.types.proxy.model_offerings import ModelOffering, ModelOfferingsConfig, SupplierConnection +from litellm.types.router import ModelMetadata @dataclass(frozen=True, slots=True) @@ -174,9 +174,7 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, if offering.source == "auto" and offering.provider_policy is None else {} ) - bundled_metadata: Final = GatewayModelMetadata.model_validate(bundled_info).model_dump( - mode="json", exclude_none=True - ) + bundled_metadata: Final = ModelMetadata.model_validate(bundled_info).model_dump(mode="json", exclude_none=True) bundled_mode: Final = bundled_info.get("mode") metadata: Final = { **bundled_metadata, @@ -190,11 +188,6 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, "model_name": offering.model_name, "litellm_params": { "model": f"{connection.provider}/{offering.upstream_model}", - **( - {"extra_body": {"provider": dict(offering.provider_policy.request_body())}} - if offering.provider_policy is not None - else {} - ), **({"api_base": connection.api_base} if connection.api_base is not None else {}), **({"api_key": connection.api_key} if connection.api_key is not None else {}), **( @@ -209,6 +202,7 @@ def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, ), }, "model_info": {**metadata, "id": deployment_id, "blocked": reason is not None}, + **({"provider_policy": offering.provider_policy} if offering.provider_policy is not None else {}), } ) @@ -315,20 +309,6 @@ class ModelOfferingsManager: unavailable, allowed_deployments, frozenset(candidate.get_model_ids()), - MappingProxyType( - { - offering.model_name: offering.provider_policy - for offering in loaded.config.offerings - if offering.model_name in available and offering.provider_policy is not None - } - ), - MappingProxyType( - { - _deployment_id(offering, providers[offering.provider].connection): offering.provider_policy - for offering in loaded.config.offerings - if offering.model_name in available and offering.provider_policy is not None - } - ), ) self.config = loaded.config self.config_fingerprint = loaded.fingerprint diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index 971524866d4..5a4de6c1a31 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -1,7 +1,7 @@ from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass, field +from dataclasses import dataclass from types import MappingProxyType from typing import Final, cast # noqa: TID251 # native Router dynamic attribute dispatch @@ -16,7 +16,6 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator, alias_target, caller_alias_maps from litellm.router import Router from litellm.router_utils.common_utils import resolve_model_group_alias -from litellm.types.proxy.model_offerings import OpenRouterProviderPolicy from litellm.types.utils import CallTypes, CallTypesLiteral @@ -27,8 +26,6 @@ class OfferingServingSnapshot: unavailable_models: Mapping[str, str] allowed_deployments: frozenset[str] allowed_deployment_ids: frozenset[str] = frozenset() - provider_policies: Mapping[str, OpenRouterProviderPolicy] = field(default_factory=lambda: MappingProxyType({})) - deployment_policies: Mapping[str, OpenRouterProviderPolicy] = field(default_factory=lambda: MappingProxyType({})) _PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] = ContextVar( @@ -50,6 +47,7 @@ _ROUTING_OVERRIDES: Final = ( "context_window_fallbacks", "content_policy_fallbacks", "model_list", + "provider_policy", "deployment_id", "specific_deployment", "model_id", @@ -240,7 +238,10 @@ class OfferingAccessGuard(CustomLogger): model="", llm_provider="", ) - if any(name in data for name in ("specific_deployment", "deployment_id", "model_id", "model_list")): + if any( + name in data + for name in ("specific_deployment", "deployment_id", "model_id", "model_list", "provider_policy") + ): return litellm.BadRequestError( message="External offering mode does not accept caller-defined deployment selection", model="", @@ -252,7 +253,8 @@ class OfferingAccessGuard(CustomLogger): snapshot: Final = self.router.serving_snapshot() offering: Final = self._offering_name(model, user_api_key_dict) if offering in snapshot.available_models: - policy: Final = snapshot.provider_policies.get(offering) + deployment: Final = snapshot.router.get_deployment_by_model_group_name(offering) + policy: Final = deployment.provider_policy if deployment is not None else None if policy is None: return None if call_type not in _POLICY_CALLS or _policy_override(data): @@ -295,7 +297,8 @@ class OfferingAccessGuard(CustomLogger): ) deployment_id: Final = model_info.get("id") snapshot: Final = self.router.serving_snapshot() - policy: Final = snapshot.deployment_policies.get(deployment_id) if isinstance(deployment_id, str) else None + deployment: Final = snapshot.router.get_deployment(deployment_id) if isinstance(deployment_id, str) else None + policy: Final = deployment.provider_policy if deployment is not None else None if policy is None and call_type not in ( CallTypes.acompletion, CallTypes.completion, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 993bfd94909..7e19b2bb926 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -160,7 +160,7 @@ from litellm.router_utils.auto_router_tuning_baseline import ( from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.router_utils.routing_groups import parse_routing_groups from litellm.types.caching import RedisPipelineIncrementOperation -from litellm.types.proxy.model_metadata import GatewayModelMetadata, resolve_gateway_model_metadata +from litellm.types.router import ModelMetadata, resolve_model_metadata from litellm.types.utils import ( PRICING_OVERRIDES_KEY, ModelResponse, @@ -14859,7 +14859,7 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } - gateway_metadata: Final = resolve_gateway_model_metadata( + gateway_metadata: Final = resolve_model_metadata( MappingProxyType({}) if llm_router is not None and getattr(llm_router, "model_metadata_authoritative", False) is True else MappingProxyType({**litellm_model_info, **discovered_model_info}), @@ -14868,11 +14868,7 @@ def _enrich_model_info_with_litellm_data( published_model: Final = { **model, "model_info": { - **{ - field: value - for field, value in model["model_info"].items() - if field not in GatewayModelMetadata.model_fields - }, + **{field: value for field, value in model["model_info"].items() if field not in ModelMetadata.model_fields}, **gateway_metadata.model_dump(mode="json", exclude_none=True), **discover_model_metadata((gateway_metadata,)), }, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f3f2dd73130..c0e968e2935 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -85,7 +85,7 @@ from litellm.proxy.model_discovery import discover_model_metadata, guaranteed_to from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.proxy.model_metadata import resolve_gateway_model_metadata +from litellm.types.router import resolve_model_metadata from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage try: @@ -9201,13 +9201,6 @@ def create_model_info_response( """ from litellm.proxy.auth.model_checks import get_all_fallbacks - base: Final[ModelInfoResponse] = { - "id": model_id, - "object": "model", - "created": DEFAULT_MODEL_CREATED_AT_TIME, - "owned_by": provider, - } - alias_target: Final = ( resolve_model_group_alias(llm_router.model_group_alias, model_id) if llm_router is not None else None ) @@ -9236,8 +9229,6 @@ def create_model_info_response( for deployment_model in deployment_models ) - max_input_tokens: int | None = _group_token_limit(candidate_sets, "max_input_tokens") - max_output_tokens: int | None = _group_token_limit(candidate_sets, "max_output_tokens") mode: Final = next( ( m @@ -9250,55 +9241,51 @@ def create_model_info_response( ), None, ) - if mode is not None: - base["mode"] = mode - - if listing_info is not None: - if listing_info.max_input_tokens is not None: - max_input_tokens = listing_info.max_input_tokens - if listing_info.max_output_tokens is not None: - max_output_tokens = listing_info.max_output_tokens - - if llm_router is not None: - configured_mode: Final = llm_router.get_configured_mode(lookup_model) - if isinstance(configured_mode, str): - base["mode"] = configured_mode - - if max_input_tokens is not None: - base["max_input_tokens"] = max_input_tokens - if max_output_tokens is not None: - base["max_output_tokens"] = max_output_tokens + configured_mode: Final = llm_router.get_configured_mode(lookup_model) if llm_router is not None else None + effective_mode: Final = configured_mode if isinstance(configured_mode, str) else mode candidates_by_model: Final = dict(zip(deployment_models, candidate_sets)) discovery_sources: Final = ( tuple( - resolve_gateway_model_metadata( + resolve_model_metadata( _merged_listing_info(candidates_by_model.get(deployment.cost_map_key, candidate_sets[0])), deployment.model_info, ) for deployment in listing_info.deployments ) if listing_info is not None and listing_info.deployments - else tuple( - resolve_gateway_model_metadata(_merged_listing_info(candidates), {}) for candidates in candidate_sets - ) + else tuple(resolve_model_metadata(_merged_listing_info(candidates), {}) for candidates in candidate_sets) ) has_context_contract: Final = any(source.context_window is not None for source in discovery_sources) - limits: Final = ( - { - field: value - for field in ("max_input_tokens", "max_output_tokens") - if (value := guaranteed_token_limit(discovery_sources, field)) is not None - } + max_input_tokens: Final = ( + guaranteed_token_limit(discovery_sources, "max_input_tokens") if has_context_contract - else {field: value for field, value in base.items() if field in ("max_input_tokens", "max_output_tokens")} + else listing_info.max_input_tokens + if listing_info is not None and listing_info.max_input_tokens is not None + else _group_token_limit(candidate_sets, "max_input_tokens") ) + max_output_tokens: Final = ( + guaranteed_token_limit(discovery_sources, "max_output_tokens") + if has_context_contract + else listing_info.max_output_tokens + if listing_info is not None and listing_info.max_output_tokens is not None + else _group_token_limit(candidate_sets, "max_output_tokens") + ) + known_limits: Final = { + field: value + for field, value in (("max_input_tokens", max_input_tokens), ("max_output_tokens", max_output_tokens)) + if value is not None + } discovered_base: Final[ModelInfoResponse] = _MODEL_LISTING_RESPONSE_ADAPTER.validate_python( { - **{field: value for field, value in base.items() if field not in ("max_input_tokens", "max_output_tokens")}, - **limits, + "id": model_id, + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": provider, + **({"mode": effective_mode} if effective_mode is not None else {}), + **known_limits, **discover_model_metadata(discovery_sources), - }, + } ) if not include_metadata: diff --git a/litellm/router.py b/litellm/router.py index f2e2067e60c..9e3d247222d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9485,7 +9485,7 @@ class Router: deployment = self._add_deployment(deployment=deployment) - model: Final = deployment.to_json(exclude_none=True) + model: Final = deployment.to_router_dict() self._add_model_to_list_and_index_map(model=model, model_id=deployment.model_info.id) return deployment @@ -10157,7 +10157,7 @@ class Router: if "*" in deployment.model_name: # store this as a regex pattern - all deployments matching this pattern will be sent to this deployment # Store deployment.model_name as a regex pattern - self.pattern_router.add_pattern(deployment.model_name, deployment.to_json(exclude_none=True)) + self.pattern_router.add_pattern(deployment.model_name, deployment.to_router_dict()) if deployment.model_info.id: self.provider_default_deployment_ids.append(deployment.model_info.id) @@ -10166,9 +10166,7 @@ class Router: if _team_id is not None and _team_public_model_name is not None and "*" in _team_public_model_name: if _team_id not in self.team_pattern_routers: self.team_pattern_routers[_team_id] = PatternMatchRouter() - self.team_pattern_routers[_team_id].add_pattern( - _team_public_model_name, deployment.to_json(exclude_none=True) - ) + self.team_pattern_routers[_team_id].add_pattern(_team_public_model_name, deployment.to_router_dict()) # Azure GPT-Vision Enhancements, users can pass os.environ/ data_sources: Final = deployment.litellm_params.get("dataSources", []) or [] @@ -10282,7 +10280,7 @@ class Router: ) # add to model list - _deployment: Final = deployment.to_json(exclude_none=True) + _deployment: Final = deployment.to_router_dict() # initialize client self._add_deployment(deployment=deployment) diff --git a/litellm/router_utils/model_request_defaults.py b/litellm/router_utils/model_request_defaults.py index f7870f8e5c1..ef457c45dd4 100644 --- a/litellm/router_utils/model_request_defaults.py +++ b/litellm/router_utils/model_request_defaults.py @@ -4,9 +4,9 @@ from typing import Final from pydantic import BaseModel, ConfigDict, TypeAdapter -from litellm.types.proxy.model_metadata import GatewayModelMetadata +from litellm.types.router import ModelMetadata -_METADATA_ADAPTER: Final = TypeAdapter(GatewayModelMetadata) +_METADATA_ADAPTER: Final = TypeAdapter(ModelMetadata) _EMPTY_DEFAULTS: Final[Mapping[str, int]] = MappingProxyType({}) _OUTPUT_BUDGET_PARAMS: Final = frozenset(("max_tokens", "max_completion_tokens", "max_output_tokens")) diff --git a/litellm/types/llms/openrouter.py b/litellm/types/llms/openrouter.py index 1558ff576b5..1f66dc3de17 100644 --- a/litellm/types/llms/openrouter.py +++ b/litellm/types/llms/openrouter.py @@ -1,7 +1,42 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Annotated, Literal + +from pydantic import ConfigDict, Field, field_validator from typing_extensions import TypedDict +from litellm.types.llms.base import LiteLLMBaseModel + class OpenRouterErrorMessage(TypedDict): message: str code: int metadata: dict + + +class OpenRouterProviderPolicy(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["openrouter"] + only: Annotated[tuple[Annotated[str, Field(min_length=1, strict=True)], ...], Field(min_length=1)] + allow_fallbacks: Literal[False] = False + + @field_validator("only") + @classmethod + def exact_endpoints(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(frozenset(value)) != len(value) or any( + endpoint != endpoint.strip() or not endpoint.strip() or "*" in endpoint or "://" in endpoint + for endpoint in value + ): + raise ValueError("Provider endpoints must be unique exact slugs, not URLs or wildcards") + return value + + @field_validator("allow_fallbacks", mode="before") + @classmethod + def no_fallbacks(cls, value: object) -> Literal[False]: + if value is not False: + raise ValueError("Offering provider policies require allow_fallbacks=false") + return False + + def request_body(self) -> Mapping[str, object]: + return MappingProxyType({"only": list(self.only), "allow_fallbacks": self.allow_fallbacks}) diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index b28c31c4449..4e6d3be01d1 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Sequence """Response types for the model listing/retrieve endpoints (/v1/models, /models).""" @@ -6,10 +6,7 @@ from typing import Literal from typing_extensions import NotRequired, ReadOnly, TypedDict - -class ModelRequestDefaultsInfo(TypedDict, total=False): - output_token_budget: ReadOnly[int] - output_token_budget_by_reasoning_effort: ReadOnly[Mapping[str, int]] +from litellm.types.utils import ModelRequestDefaultsInfo class ModelDiscoveryInfo(TypedDict, total=False): diff --git a/litellm/types/proxy/model_metadata.py b/litellm/types/proxy/model_metadata.py index 21478df2c1b..92783573743 100644 --- a/litellm/types/proxy/model_metadata.py +++ b/litellm/types/proxy/model_metadata.py @@ -1,128 +1,11 @@ -from collections.abc import Mapping, Sequence -from types import MappingProxyType -from typing import Annotated, Final, cast # noqa: TID251 # Pydantic runtime schema boundary - -from pydantic import ( - ConfigDict, - Field, - TypeAdapter, - ValidationError, - ValidationInfo, - field_serializer, - field_validator, +from litellm.types.router import ( + ModelMetadata as GatewayModelMetadata, +) +from litellm.types.router import ( + ModelRequestDefaults, +) +from litellm.types.router import ( + resolve_model_metadata as resolve_gateway_model_metadata, ) -from litellm.types.llms.base import LiteLLMBaseModel - -_STRING_VALUES_ADAPTER: Final = TypeAdapter(tuple[str, ...]) -_VALIDATED_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) - - -class ModelRequestDefaults(LiteLLMBaseModel): - model_config = ConfigDict(extra="forbid", frozen=True) - - output_token_budget: Annotated[int, Field(gt=0, strict=True)] | None = None - output_token_budget_by_reasoning_effort: Mapping[str, Annotated[int, Field(gt=0, strict=True)]] | None = None - - @field_validator("output_token_budget_by_reasoning_effort") - @classmethod - def immutable_budgets(cls, value: Mapping[str, int] | None) -> Mapping[str, int] | None: - return MappingProxyType(dict(value)) if value is not None else None - - @field_serializer("output_token_budget_by_reasoning_effort") - def serialize_budgets(self, value: Mapping[str, int] | None) -> Mapping[str, int] | None: - return dict(value) if value is not None else None - - def output_budget(self, effort: str | None) -> int | None: - return ( - (self.output_token_budget_by_reasoning_effort or {}).get(effort, self.output_token_budget) - if effort is not None - else self.output_token_budget - ) - - -class GatewayModelMetadata(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore", frozen=True) - - context_window: int | None = Field( - default=None, gt=0, description="Maximum combined input and generated output tokens" - ) - max_input_tokens: int | None = Field(default=None, gt=0, description="Independent maximum input tokens") - max_output_tokens: int | None = Field(default=None, gt=0, description="Independent maximum generated output tokens") - supports_function_calling: bool | None = None - supports_parallel_function_calling: bool | None = None - supports_reasoning: bool | None = None - reasoning_effort_levels: Sequence[str] | None = Field( - default=None, description="Exact effort values this route accepts" - ) - default_reasoning_effort: str | None = Field(default=None, description="Effort used when the request omits it") - supported_endpoints: Sequence[str] | None = Field( - default=None, description="Inference endpoints exposed by this route" - ) - supported_modalities: Sequence[str] | None = None - supported_output_modalities: Sequence[str] | None = None - request_defaults: ModelRequestDefaults | None = Field( - default=None, description="Configured request defaults, independent of supplier capability limits" - ) - - @classmethod - def validate_field_input(cls, field_name: str, value: object) -> None: - TypeAdapter[object]( - cast(object, cls.model_fields[field_name].rebuild_annotation()) # cast-ok: runtime Pydantic field schema - ).validate_python(value, strict=True) - - @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") - @classmethod - def positive_limit(cls, value: object) -> int | None: - return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None - - @field_validator( - "supports_function_calling", "supports_parallel_function_calling", "supports_reasoning", mode="before" - ) - @classmethod - def known_boolean(cls, value: object) -> bool | None: - return value if isinstance(value, bool) else None - - @field_validator( - "reasoning_effort_levels", - "supported_endpoints", - "supported_modalities", - "supported_output_modalities", - mode="before", - ) - @classmethod - def known_string_list(cls, value: object) -> Sequence[str] | None: - try: - return tuple(dict.fromkeys(_STRING_VALUES_ADAPTER.validate_python(value))) - except ValidationError: - return None - - @field_validator("default_reasoning_effort", mode="before") - @classmethod - def known_string(cls, value: object) -> str | None: - return value if isinstance(value, str) else None - - @field_validator("reasoning_effort_levels") - @classmethod - def enabled_reasoning_levels(cls, value: Sequence[str] | None, info: ValidationInfo) -> Sequence[str] | None: - data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) - return None if data.get("supports_reasoning") is False else value - - @field_validator("default_reasoning_effort") - @classmethod - def supported_default(cls, value: str | None, info: ValidationInfo) -> str | None: - data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) - levels: Final = data.get("reasoning_effort_levels") - if data.get("supports_reasoning") is False: - return None - return value if levels is None or isinstance(levels, (list, tuple)) and value in levels else None - - -def resolve_gateway_model_metadata( - catalog: Mapping[str, object], configured: Mapping[str, object] -) -> GatewayModelMetadata: - values: Final = { - field: configured[field] if configured.get(field) is not None else catalog.get(field) - for field in GatewayModelMetadata.model_fields - } - return GatewayModelMetadata.model_validate(values) +__all__ = ("GatewayModelMetadata", "ModelRequestDefaults", "resolve_gateway_model_metadata") diff --git a/litellm/types/proxy/model_offerings.py b/litellm/types/proxy/model_offerings.py index 1c1794fc5e9..0d6488ae07a 100644 --- a/litellm/types/proxy/model_offerings.py +++ b/litellm/types/proxy/model_offerings.py @@ -13,10 +13,11 @@ from pydantic import ( model_validator, ) -from litellm.types.proxy.model_metadata import GatewayModelMetadata +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.router import ModelMetadata -class OfferingModelInfo(GatewayModelMetadata): +class OfferingModelInfo(ModelMetadata): model_config = ConfigDict(extra="forbid", frozen=True) mode: Literal["chat", "completion", "embedding"] | None = None @@ -52,34 +53,6 @@ class SupplierConnection(BaseModel): return self -class OpenRouterProviderPolicy(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True) - - provider: Literal["openrouter"] - only: Annotated[tuple[Annotated[str, Field(min_length=1, strict=True)], ...], Field(min_length=1)] - allow_fallbacks: Literal[False] = False - - @field_validator("only") - @classmethod - def exact_endpoints(cls, value: tuple[str, ...]) -> tuple[str, ...]: - if len(frozenset(value)) != len(value) or any( - endpoint != endpoint.strip() or not endpoint.strip() or "*" in endpoint or "://" in endpoint - for endpoint in value - ): - raise ValueError("Provider endpoints must be unique exact slugs, not URLs or wildcards") - return value - - @field_validator("allow_fallbacks", mode="before") - @classmethod - def no_fallbacks(cls, value: object) -> Literal[False]: - if value is not False: - raise ValueError("Offering provider policies require allow_fallbacks=false") - return False - - def request_body(self) -> Mapping[str, object]: - return MappingProxyType({"only": list(self.only), "allow_fallbacks": self.allow_fallbacks}) - - class ModelOffering(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) diff --git a/litellm/types/router.py b/litellm/types/router.py index 90e2b5a53a0..ba403a060c0 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,6 +6,7 @@ import datetime import enum from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -16,19 +17,30 @@ from typing import ( Literal, TypeAlias, TypeVar, + cast, # noqa: TID251 # runtime Pydantic field schema boundary get_type_hints, ) from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout -from pydantic import ConfigDict, Field, JsonValue, field_validator, model_validator +from pydantic import ( + ConfigDict, + Field, + JsonValue, + TypeAdapter, + ValidationError, + ValidationInfo, + field_serializer, + field_validator, + model_validator, +) from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.completion_timeout import CompletionTimeout -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import coerce_token_limit, normalize_drop_params from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router_weights import RouterWeights @@ -41,6 +53,7 @@ from .embedding import EmbeddingRequest from .litellm_params import RoutingStrategyName from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject +from .llms.openrouter import OpenRouterProviderPolicy from .search import SearchProvider from .utils import ( CustomPricingLiteLLMParams, @@ -238,7 +251,149 @@ class ModelAccessWindow(LiteLLMBaseModel): return self -class ModelInfo(MirroredPricingParams): +_STRING_VALUES_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_VALIDATED_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +class ModelRequestDefaults(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + output_token_budget: Annotated[int, Field(gt=0, strict=True)] | None = Field( + default=None, description="Default requested output budget, not a physical model limit" + ) + output_token_budget_by_reasoning_effort: Mapping[str, Annotated[int, Field(gt=0, strict=True)]] | None = Field( + default=None, description="Requested output budgets keyed by exact reasoning effort" + ) + + @field_validator("output_token_budget_by_reasoning_effort") + @classmethod + def immutable_budgets(cls, value: Mapping[str, int] | None) -> Mapping[str, int] | None: + return MappingProxyType(dict(value)) if value is not None else None + + @field_serializer("output_token_budget_by_reasoning_effort") + def serialize_budgets(self, value: Mapping[str, int] | None) -> Mapping[str, int] | None: + return dict(value) if value is not None else None + + def output_budget(self, effort: str | None) -> int | None: + return ( + (self.output_token_budget_by_reasoning_effort or {}).get(effort, self.output_token_budget) + if effort is not None + else self.output_token_budget + ) + + +class ModelMetadata(LiteLLMBaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + context_window: int | None = Field( + default=None, gt=0, description="Maximum combined input and generated output tokens" + ) + max_input_tokens: int | None = Field(default=None, gt=0, description="Independent maximum input tokens") + max_output_tokens: int | None = Field(default=None, gt=0, description="Independent maximum generated output tokens") + supports_function_calling: bool | None = Field(default=None, description="Whether this model accepts tool calls") + supports_parallel_function_calling: bool | None = Field( + default=None, description="Whether this model accepts parallel tool calls" + ) + supports_reasoning: bool | None = Field(default=None, description="Whether this model supports reasoning controls") + reasoning_effort_levels: Sequence[str] | None = Field( + default=None, description="Exact effort values this route accepts" + ) + default_reasoning_effort: str | None = Field(default=None, description="Effort used when the request omits it") + supported_endpoints: Sequence[str] | None = Field( + default=None, description="Inference endpoints exposed by this route" + ) + supported_modalities: Sequence[str] | None = Field(default=None, description="Accepted input modalities") + supported_output_modalities: Sequence[str] | None = Field(default=None, description="Supported output modalities") + request_defaults: ModelRequestDefaults | None = Field( + default=None, description="Configured request defaults, independent of supplier capability limits" + ) + + @classmethod + def validate_field_input(cls, field_name: str, value: object) -> None: + TypeAdapter[object]( + cast(object, cls.model_fields[field_name].rebuild_annotation()) # cast-ok: runtime Pydantic field schema + ).validate_python(value, strict=True) + + @classmethod + def validate_supplier_field(cls, field_name: str | None, value: object) -> object: + normalized: Final = ( + None + if field_name in ("context_window", "max_input_tokens", "max_output_tokens") + and type(value) is int + and value == 0 + else value + ) + if field_name is not None and field_name in cls.model_fields: + cls.validate_field_input(field_name, normalized) + return normalized + + @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") + @classmethod + def positive_limit(cls, value: object, _info: ValidationInfo) -> int | None: + return coerce_token_limit(value, strict_positive=True) + + @field_validator( + "supports_function_calling", "supports_parallel_function_calling", "supports_reasoning", mode="before" + ) + @classmethod + def known_boolean(cls, value: object) -> bool | None: + return value if isinstance(value, bool) else None + + @field_validator( + "reasoning_effort_levels", + "supported_endpoints", + "supported_modalities", + "supported_output_modalities", + mode="before", + ) + @classmethod + def known_string_list(cls, value: object) -> Sequence[str] | None: + try: + return tuple(dict.fromkeys(_STRING_VALUES_ADAPTER.validate_python(value))) + except ValidationError: + return None + + @field_validator("default_reasoning_effort", mode="before") + @classmethod + def known_string(cls, value: object) -> str | None: + return value if isinstance(value, str) else None + + @field_validator("reasoning_effort_levels") + @classmethod + def enabled_reasoning_levels(cls, value: Sequence[str] | None, info: ValidationInfo) -> Sequence[str] | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) + return None if data.get("supports_reasoning") is False else value + + @field_validator("default_reasoning_effort") + @classmethod + def supported_default(cls, value: str | None, info: ValidationInfo) -> str | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) + levels: Final = data.get("reasoning_effort_levels") + if data.get("supports_reasoning") is False: + return None + return value if levels is None or isinstance(levels, (list, tuple)) and value in levels else None + + +def resolve_model_metadata(catalog: Mapping[str, object], configured: Mapping[str, object]) -> ModelMetadata: + values: Final = { + field: configured[field] if configured.get(field) is not None else catalog.get(field) + for field in ModelMetadata.model_fields + } + return ModelMetadata.model_validate(values) + + +class ModelInfo(ModelMetadata, MirroredPricingParams): + max_input_tokens: int | None = Field(default=None, description="Independent maximum input tokens") + max_output_tokens: int | None = Field(default=None, description="Independent maximum generated output tokens") + + @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") + @classmethod + def positive_limit(cls, value: object, _info: ValidationInfo) -> int | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(_info.data) + return coerce_token_limit( + value, strict_positive=_info.field_name == "context_window" or data.get("context_window") is not None + ) + id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config. updated_at: datetime.datetime | None = None @@ -320,7 +475,7 @@ class ModelInfo(MirroredPricingParams): raise ValueError("ptu_effective_to must be after ptu_effective_from") return self - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=False) def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator @@ -753,6 +908,7 @@ class DeploymentTypedDict(TypedDict, total=False): model_name: Required[str] litellm_params: Required[LiteLLMParamsTypedDict] model_info: dict + provider_policy: ReadOnly[OpenRouterProviderPolicy] SPECIAL_MODEL_INFO_PARAMS = tuple(MirroredPricingParams.model_fields) @@ -762,6 +918,9 @@ class Deployment(LiteLLMBaseModel): model_name: str litellm_params: LiteLLM_Params model_info: ModelInfo + provider_policy: OpenRouterProviderPolicy | None = Field( + default=None, exclude=True, description="Gateway-owned supplier selection, excluded from public model metadata" + ) # admin-toggled pause flag; mirrors LiteLLM_ProxyModelTable.blocked. None means "don't set it # on create" -- the Prisma column defaults to False -- rather than "explicitly unblocked". blocked: bool | None = None @@ -799,6 +958,27 @@ class Deployment(LiteLLMBaseModel): # if using pydantic v1 return self.dict(**kwargs) + def to_router_dict(self) -> dict[str, object]: # mutable-ok: native Router stores mutable deployment records + return { + **self.model_dump(exclude_none=True), + **({"provider_policy": self.provider_policy} if self.provider_policy is not None else {}), + } + + @model_validator(mode="after") + def apply_provider_policy(self) -> "Deployment": + policy: Final = self.provider_policy + if policy is None: + return self + provider: Final = self.litellm_params.custom_llm_provider or self.litellm_params.model.partition("/")[0] + if provider != policy.provider: + raise ValueError("The deployment provider must match its server-owned provider policy") + body: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(self.litellm_params.get("extra_body") or {}) + routing: Final = dict(policy.request_body()) + if body.get("provider") is not None and body["provider"] != routing: + raise ValueError("The deployment routing body conflicts with its server-owned provider policy") + self.litellm_params = self.litellm_params.model_copy(update={"extra_body": {**body, "provider": routing}}) + return self + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d422f33f771..20d891a0854 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -283,10 +283,16 @@ class OffPeakPricing(TypedDict, total=False): cache_creation_input_token_cost: ReadOnly[float] +class ModelRequestDefaultsInfo(TypedDict, total=False): + output_token_budget: ReadOnly[int] + output_token_budget_by_reasoning_effort: ReadOnly[Mapping[str, int]] + + class ModelInfoBase(ProviderSpecificModelInfo, total=False): context_window: ReadOnly[int | None] supported_modalities: ReadOnly[Sequence[str] | None] supported_output_modalities: ReadOnly[Sequence[str] | None] + request_defaults: ReadOnly[ModelRequestDefaultsInfo | None] key: Required[str] # the key in litellm.model_cost which is returned max_tokens: Required[int | None] diff --git a/litellm/utils.py b/litellm/utils.py index e11bb3ab8a0..ae8a96bdd1e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6205,6 +6205,7 @@ def get_model_info_helper( context_window=_model_info.get("context_window", None), supported_modalities=_model_info.get("supported_modalities", None), supported_output_modalities=_model_info.get("supported_output_modalities", None), + request_defaults=_model_info.get("request_defaults", None), max_tokens=_model_info.get("max_tokens", None), max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), diff --git a/tests/unit/llms/test_model_inventory.py b/tests/unit/llms/test_model_inventory.py index 336824f7705..069f2615a15 100644 --- a/tests/unit/llms/test_model_inventory.py +++ b/tests/unit/llms/test_model_inventory.py @@ -2,9 +2,70 @@ import json from pathlib import Path from typing import Final +import httpx import pytest -from litellm.llms.model_inventory import model_inventory_api_base, model_inventory_identity +import litellm +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.model_inventory import ( + get_provider_model_inventory, + model_inventory_api_base, + model_inventory_identity, +) +from litellm.llms.openrouter.responses.transformation import OpenRouterResponsesAPIConfig +from litellm.llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig +from litellm.types.proxy.model_inventory import SupplierModelInventory + + +@pytest.mark.parametrize("provider", ("openrouter", "vercel_ai_gateway")) +@pytest.mark.parametrize("source", ("explicit", "environment", "global", "default")) +def test_inventory_base_matches_native_resolution_and_preserves_paths( + monkeypatch: pytest.MonkeyPatch, provider: str, source: str +) -> None: + explicit: Final = "https://explicit.test/custom/v1/" if source == "explicit" else None + environment: Final = "https://environment.test/custom/v1" + global_base: Final = "https://global.test/custom/v1" + monkeypatch.setattr(litellm, "api_base", global_base if source in ("explicit", "global") else None) + variable: Final = "OPENROUTER_API_BASE" if provider == "openrouter" else "VERCEL_AI_GATEWAY_API_BASE" + if source in ("explicit", "environment"): + monkeypatch.setenv(variable, environment) + else: + monkeypatch.delenv(variable, raising=False) + expected: Final = ( + explicit or OpenRouterResponsesAPIConfig().get_complete_url(None, {}).removesuffix("/responses") + if provider == "openrouter" + else VercelAIGatewayConfig().get_openai_compatible_provider_info(explicit, "fixture-key")[0] + ) + assert model_inventory_api_base(provider, explicit) == expected + + +@pytest.mark.parametrize("provider", ("openrouter", "vercel_ai_gateway")) +async def test_native_base_resolution_does_not_borrow_ambient_discovery_credentials( + monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + base: Final = "https://supplier.test/custom/v1" + variable: Final = "OPENROUTER" if provider == "openrouter" else "VERCEL_AI_GATEWAY" + monkeypatch.setenv(f"{variable}_API_BASE", base) + monkeypatch.setenv(f"{variable}_API_KEY", "ambient-provider-key") + monkeypatch.setenv("VERCEL_OIDC_TOKEN", "ambient-oidc") + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "api_key", "ambient-global-key") + + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == f"{base}/models" + assert "authorization" not in request.headers + return httpx.Response(200, json={"data": [{"id": "org/backend"}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + inventory: Final = await get_provider_model_inventory( + provider=provider, api_base=None, headers={}, client=handler, cache=InMemoryCache() + ) + assert isinstance(inventory, SupplierModelInventory) + assert frozenset(inventory.models) == {"org/backend"} @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index a751d27f320..312b2a073ff 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -23,6 +23,67 @@ from litellm.router import Router from litellm.types.utils import CallTypesLiteral +@pytest.mark.parametrize("provider", ("openrouter", "vercel_ai_gateway")) +async def test_native_base_changes_invalidate_connection_inventory_and_preserve_pinned_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + variable: Final = "OPENROUTER" if provider == "openrouter" else "VERCEL_AI_GATEWAY" + first_base: Final = "https://first.test/custom/v1" + second_base: Final = "https://second.test/custom/v1" + monkeypatch.setenv(f"{variable}_API_BASE", first_base) + monkeypatch.setenv(f"{variable}_API_KEY", "ambient-provider-key") + monkeypatch.setattr(litellm, "api_base", None) + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": {"supplier": {"provider": provider, "api_key": "configured-key"}}, + "offerings": [ + { + "model_name": "selected", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + } + ], + } + ) + ) + responses: Final = iter( + ( + (first_base, 200), + (second_base, 401), + (second_base, 200), + ) + ) + + def respond(request: httpx.Request) -> httpx.Response: + expected_base, status = next(responses) + assert str(request.url) == f"{expected_base}/models" + assert request.headers["authorization"] == "Bearer configured-key" + return httpx.Response(status, json={"data": [{"id": "org/backend"}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + original_ids: Final = manager.router.get_model_ids() + with manager.router.pin_snapshot(): + monkeypatch.setenv(f"{variable}_API_BASE", second_base) + assert await manager.reload() + assert manager.router.get_model_ids() == original_ids + assert manager.router.latest_snapshot().available_models == set() + assert manager.router.get_model_names() == [] + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + assert manager.router.get_model_ids() != original_ids + + @pytest.mark.parametrize("catalog_state", ("missing", "known", "different_route_provider")) async def test_wildcard_listing_tracks_selected_offerings_and_pinned_reload( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, catalog_state: str @@ -993,7 +1054,10 @@ async def test_endpoint_policy_reload_isolates_aliases_identity_and_metadata( path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 ) assert await manager.reload(initial=True) - before: Final = manager.router.latest_snapshot() + selected_deployment: Final = manager.router.get_deployment_by_model_group_name("selected") + assert selected_deployment is not None + assert selected_deployment.provider_policy.only == ("synthetic/fp8",) + assert "provider_policy" not in selected_deployment.model_dump() selected_id: Final = manager.router.get_model_ids("selected") ordinary_id: Final = manager.router.get_model_ids("ordinary") pinned_metadata: Final = create_model_info_response("selected", "openrouter", llm_router=manager.router) @@ -1020,12 +1084,15 @@ async def test_endpoint_policy_reload_isolates_aliases_identity_and_metadata( ) ) assert await manager.reload() - assert manager.router.serving_snapshot().provider_policies == before.provider_policies + assert ( + manager.router.get_deployment_by_model_group_name("selected").provider_policy + == selected_deployment.provider_policy + ) assert manager.router.get_model_ids("selected") == selected_id assert manager.router.get_model_ids("selected") != selected_id assert manager.router.get_model_ids("ordinary") == ordinary_id - assert manager.router.latest_snapshot().provider_policies["selected"].only == ("synthetic/bf16",) - assert "ordinary" not in manager.router.latest_snapshot().provider_policies + assert manager.router.get_deployment_by_model_group_name("selected").provider_policy.only == ("synthetic/bf16",) + assert manager.router.get_deployment_by_model_group_name("ordinary").provider_policy is None valid: Final = manager.router.latest_snapshot() path.write_text( yaml.safe_dump( diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index 263c8d3c08e..bc37d19617c 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -26,6 +26,30 @@ from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes +def _policy_snapshot(policy: OpenRouterProviderPolicy) -> OfferingServingSnapshot: + native: Final = Router( + model_list=[ + { + "model_name": "selected", + "litellm_params": { + "model": "openrouter/backend", + "extra_body": {"provider": dict(policy.request_body())}, + }, + "model_info": {"id": "selected-id"}, + "provider_policy": policy, + }, + {"model_name": "ordinary", "litellm_params": {"model": "openai/ordinary", "api_key": "fixture-key"}}, + ] + ) + return OfferingServingSnapshot( + native, + frozenset({"selected", "ordinary"}), + MappingProxyType({}), + frozenset({"openrouter/backend", "openai/ordinary"}), + frozenset(native.get_model_ids()), + ) + + async def test_asgi_stream_pins_authority_until_final_chunk_and_passes_lifespan_through() -> None: first: Final = OfferingServingSnapshot( Router(model_list=[]), frozenset({"first"}), MappingProxyType({}), frozenset() @@ -251,6 +275,8 @@ async def test_body_model_override_cannot_bypass_offering_authority() -> None: @pytest.mark.parametrize( "override", ( + {"provider_policy": {"provider": "openrouter", "only": ["other"]}}, + {"extra_body": {"provider_policy": {"provider": "openrouter", "only": ["other"]}}}, {"provider": {"only": ["other"]}}, {"extra_body": {"provider": {"allow_fallbacks": True}}}, {"extra_body": {"extra_body": {"provider": {"only": ["other"]}}}}, @@ -276,15 +302,7 @@ async def test_body_model_override_cannot_bypass_offering_authority() -> None: ) async def test_pinned_policy_rejects_caller_routing_even_through_alias(override: dict[str, object]) -> None: policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) - view: Final = OfferingRouterView( - OfferingServingSnapshot( - Router(model_list=[]), - frozenset({"selected", "ordinary"}), - MappingProxyType({}), - frozenset(), - provider_policies=MappingProxyType({"selected": policy}), - ) - ) + view: Final = OfferingRouterView(_policy_snapshot(policy)) guard: Final = OfferingAccessGuard(view) auth: Final = UserAPIKeyAuth(aliases={"alias": "selected"}) for call_type in ("acompletion", "aresponses", "anthropic_messages"): @@ -295,15 +313,7 @@ async def test_pinned_policy_rejects_caller_routing_even_through_alias(override: async def test_pinned_policy_shapes_requests_without_mutation_and_blocks_unsupported_apis() -> None: policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) - view: Final = OfferingRouterView( - OfferingServingSnapshot( - Router(model_list=[]), - frozenset({"selected", "ordinary"}), - MappingProxyType({}), - frozenset(), - provider_policies=MappingProxyType({"selected": policy}), - ) - ) + view: Final = OfferingRouterView(_policy_snapshot(policy)) guard: Final = OfferingAccessGuard(view) original: Final = {"model": "selected", "extra_body": {"temperature": 0.5}} assert await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), original, "acompletion") == { @@ -333,16 +343,7 @@ async def test_pinned_policy_shapes_requests_without_mutation_and_blocks_unsuppo @pytest.mark.parametrize("call_type", (None, CallTypes.aresponses_websocket, CallTypes.aimage_generation)) async def test_pinned_deployment_rejects_apis_that_cannot_carry_policy(call_type: CallTypes | None) -> None: policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) - view: Final = OfferingRouterView( - OfferingServingSnapshot( - Router(model_list=[]), - frozenset({"selected"}), - MappingProxyType({}), - frozenset({"openrouter/backend"}), - frozenset({"selected-id"}), - deployment_policies=MappingProxyType({"selected-id": policy}), - ) - ) + view: Final = OfferingRouterView(_policy_snapshot(policy)) with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): await OfferingAccessGuard(view).async_pre_call_deployment_hook( { diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index d7512fefea1..33f82d3c753 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -727,3 +727,32 @@ def test_schema_and_model_info_preserve_independent_total_context(committed_sche 200, 200, ) + + +def test_native_model_info_and_lookup_preserve_reasoning_capabilities_and_request_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.types.router import ModelInfo + + metadata: Final = { + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "supports_function_calling": True, + "supports_parallel_function_calling": False, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "request_defaults": { + "output_token_budget": 100, + "output_token_budget_by_reasoning_effort": {"max": 180}, + }, + } + native: Final = ModelInfo(**metadata) + assert native.request_defaults.output_budget("max") == 180 + monkeypatch.setattr(litellm, "model_cost", {"fixture-model": {"litellm_provider": "openai", **metadata}}) + info: Final = litellm.get_model_info("fixture-model") + assert {field: info.get(field) for field in metadata} == metadata diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 6188bb966ee..4774609f594 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -29,6 +29,67 @@ from litellm.types.utils import ( ) +@pytest.mark.parametrize("value,expected", (("1200", 1200), (1.5, 1), (0, 0), (-1, -1), (True, None), ("bad", None))) +def test_native_model_info_preserves_legacy_limits_without_a_context_contract( + value: object, expected: int | None +) -> None: + info: Final = ModelInfo(max_input_tokens=value) + assert info.max_input_tokens == expected + + +@pytest.mark.parametrize("value", ("1200", 1.5, 0, -1, True, "bad")) +def test_native_context_contract_does_not_invent_invalid_input_capacity(value: object) -> None: + info: Final = ModelInfo(context_window=2000, max_input_tokens=value) + assert info.context_window == 2000 + assert info.max_input_tokens is None + + +def test_native_deployment_compiles_private_policy_and_keeps_it_separate_from_mutable_request_body() -> None: + from litellm.router import Router + from litellm.types.llms.openrouter import OpenRouterProviderPolicy + + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + native: Final = Router( + model_list=[ + { + "model_name": "public-model", + "litellm_params": {"model": "openrouter/org/backend", "extra_body": {"temperature": 0.5}}, + "model_info": {"id": "private-deployment"}, + "provider_policy": policy, + } + ] + ) + deployment: Final = native.get_deployment("private-deployment") + assert deployment is not None + assert deployment.provider_policy == policy + assert deployment.litellm_params.extra_body == { + "temperature": 0.5, + "provider": {"only": ["synthetic/fp8"], "allow_fallbacks": False}, + } + assert "provider_policy" not in deployment.model_dump() + assert "provider_policy" not in deployment.model_info.model_dump() + deployment.litellm_params.extra_body["provider"]["only"].append("caller/override") + assert native.get_deployment("private-deployment").provider_policy.only == ("synthetic/fp8",) + + +@pytest.mark.parametrize( + "model,body", + ( + ("openai/org/backend", {}), + ("openrouter/org/backend", {"provider": {"only": ["other"], "allow_fallbacks": True}}), + ), +) +def test_native_deployment_rejects_policy_provider_or_body_conflicts(model: str, body: dict[str, object]) -> None: + from litellm.types.llms.openrouter import OpenRouterProviderPolicy + + with pytest.raises(ValidationError, match=r"provider|routing body"): + Deployment( + model_name="public-model", + litellm_params=LiteLLM_Params(model=model, extra_body=body), + provider_policy=OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)), + ) + + def test_sdk_timeout_is_normalized_for_provider_clients() -> None: timeout: Final = SDKTimeout(connect=2.0, read=None, write=5.0, pool=7.0) params: Final = GenericLiteLLMParams(timeout=timeout) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0ebf2fd0c51..bfe94b250e8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31368,6 +31368,8 @@ export interface components { model_info: components["schemas"]["litellm__types__router__ModelInfo"]; /** Model Name */ model_name: string; + /** @description Gateway-owned supplier selection, excluded from public model metadata */ + provider_policy?: components["schemas"]["OpenRouterProviderPolicy"] | null; } & { [key: string]: unknown; }; @@ -37961,9 +37963,15 @@ export interface components { }; /** ModelRequestDefaults */ ModelRequestDefaults: { - /** Output Token Budget */ + /** + * Output Token Budget + * @description Default requested output budget, not a physical model limit + */ output_token_budget?: number | null; - /** Output Token Budget By Reasoning Effort */ + /** + * Output Token Budget By Reasoning Effort + * @description Requested output budgets keyed by exact reasoning effort + */ output_token_budget_by_reasoning_effort?: { [key: string]: number; } | null; @@ -39212,6 +39220,22 @@ export interface components { */ type: "openIdConnect"; }; + /** OpenRouterProviderPolicy */ + OpenRouterProviderPolicy: { + /** + * Allow Fallbacks + * @default false + * @constant + */ + allow_fallbacks: false; + /** Only */ + only: string[]; + /** + * Provider + * @constant + */ + provider: "openrouter"; + }; /** OpenSourceClassifierConfig */ OpenSourceClassifierConfig: { /** @@ -49431,15 +49455,30 @@ export interface components { * @description Inference endpoints exposed by this route */ supported_endpoints?: string[] | null; - /** Supported Modalities */ + /** + * Supported Modalities + * @description Accepted input modalities + */ supported_modalities?: string[] | null; - /** Supported Output Modalities */ + /** + * Supported Output Modalities + * @description Supported output modalities + */ supported_output_modalities?: string[] | null; - /** Supports Function Calling */ + /** + * Supports Function Calling + * @description Whether this model accepts tool calls + */ supports_function_calling?: boolean | null; - /** Supports Parallel Function Calling */ + /** + * Supports Parallel Function Calling + * @description Whether this model accepts parallel tool calls + */ supports_parallel_function_calling?: boolean | null; - /** Supports Reasoning */ + /** + * Supports Reasoning + * @description Whether this model supports reasoning controls + */ supports_reasoning?: boolean | null; } & { [key: string]: unknown; @@ -49469,6 +49508,11 @@ export interface components { cache_creation_input_token_cost?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; + /** + * Context Window + * @description Maximum combined input and generated output tokens + */ + context_window?: number | null; /** Cost Per Ptu Per Hour */ cost_per_ptu_per_hour?: number | null; /** Created At */ @@ -49480,6 +49524,11 @@ export interface components { * @default false */ db_model: boolean; + /** + * Default Reasoning Effort + * @description Effort used when the request omits it + */ + default_reasoning_effort?: string | null; /** Discoverable */ discoverable?: boolean | null; /** Enable Tag Filtering */ @@ -49492,6 +49541,16 @@ export interface components { input_cost_per_token?: number | null; /** Internal Router Model */ internal_router_model?: boolean | null; + /** + * Max Input Tokens + * @description Independent maximum input tokens + */ + max_input_tokens?: number | null; + /** + * Max Output Tokens + * @description Independent maximum generated output tokens + */ + max_output_tokens?: number | null; /** * Member Auto Router * @default false @@ -49507,6 +49566,43 @@ export interface components { ptu_effective_from?: string | null; /** Ptu Effective To */ ptu_effective_to?: string | null; + /** + * Reasoning Effort Levels + * @description Exact effort values this route accepts + */ + reasoning_effort_levels?: string[] | null; + /** @description Configured request defaults, independent of supplier capability limits */ + request_defaults?: components["schemas"]["ModelRequestDefaults"] | null; + /** + * Supported Endpoints + * @description Inference endpoints exposed by this route + */ + supported_endpoints?: string[] | null; + /** + * Supported Modalities + * @description Accepted input modalities + */ + supported_modalities?: string[] | null; + /** + * Supported Output Modalities + * @description Supported output modalities + */ + supported_output_modalities?: string[] | null; + /** + * Supports Function Calling + * @description Whether this model accepts tool calls + */ + supports_function_calling?: boolean | null; + /** + * Supports Parallel Function Calling + * @description Whether this model accepts parallel tool calls + */ + supports_parallel_function_calling?: boolean | null; + /** + * Supports Reasoning + * @description Whether this model supports reasoning controls + */ + supports_reasoning?: boolean | null; /** Team Id */ team_id?: string | null; /** Team Public Model Name */ From 2a7ffd25e50270952e2f824e15f35c53d9a6da6d Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Tue, 13 Oct 2026 18:33:03 +0200 Subject: [PATCH 24/34] refactor(proxy): remove obsolete metadata compatibility wrappers --- litellm/llms/chatgpt/model_info.py | 19 ------ litellm/llms/model_inventory.py | 10 +--- litellm/proxy/model_offerings.py | 4 +- litellm/proxy/offering_router.py | 4 +- litellm/types/proxy/model_metadata.py | 11 ---- tests/unit/llms/chatgpt/test_model_info.py | 59 ++++++++----------- tests/unit/proxy/test_model_discovery.py | 12 ++-- tests/unit/proxy/test_offering_router.py | 2 +- .../test_model_request_defaults.py | 10 ++-- .../unit/types/proxy/test_model_offerings.py | 3 +- 10 files changed, 45 insertions(+), 89 deletions(-) delete mode 100644 litellm/types/proxy/model_metadata.py diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py index c88d8f96db0..5d41fd7656f 100644 --- a/litellm/llms/chatgpt/model_info.py +++ b/litellm/llms/chatgpt/model_info.py @@ -24,7 +24,6 @@ from .authenticator import Authenticator from .common_utils import get_chatgpt_default_headers _CATALOG_CLIENT_VERSION: Final = "0.159.3" -_EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) class _ReasoningLevel(BaseModel): @@ -146,24 +145,6 @@ def chatgpt_model_inventory_identity() -> str | None: return None -async def get_chatgpt_model_info( - *, - model: str, - client: AsyncHTTPHandler, - cache: InMemoryCache, - api_base: str | None = None, - authenticator: Authenticator | None = None, -) -> Mapping[str, object]: - inventory: Final = await get_chatgpt_model_inventory( - client=client, cache=cache, api_base=api_base, authenticator=authenticator - ) - return ( - inventory.models.get(model, _EMPTY_METADATA) - if isinstance(inventory, SupplierModelInventory) - else _EMPTY_METADATA - ) - - async def get_chatgpt_model_inventory( *, client: AsyncHTTPHandler, diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py index 89551a794a1..76fd286c525 100644 --- a/litellm/llms/model_inventory.py +++ b/litellm/llms/model_inventory.py @@ -1,7 +1,6 @@ """Supplier-specific inventory dispatch shared by router and selected-offering discovery.""" -from collections.abc import Generator, Mapping -from contextlib import contextmanager +from collections.abc import Mapping from types import MappingProxyType from typing import Final @@ -9,7 +8,6 @@ from pydantic import TypeAdapter import litellm from litellm.caching.in_memory_cache import InMemoryCache -from litellm.llms.chatgpt.authenticator import prevent_device_login from litellm.llms.chatgpt.model_info import ( chatgpt_model_inventory_api_base, chatgpt_model_inventory_identity, @@ -24,12 +22,6 @@ _EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) _HEADERS: Final = TypeAdapter(Mapping[str, str]) -@contextmanager -def prevent_supplier_device_login() -> Generator[None]: - with prevent_device_login(): - yield - - def model_inventory_api_base(provider: str, api_base: str | None) -> str | None: if provider == "chatgpt": return chatgpt_model_inventory_api_base(api_base) diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index 25c79c3067d..1ff3d904f98 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -14,12 +14,12 @@ import yaml from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.llms.chatgpt.authenticator import prevent_device_login from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.model_inventory import ( get_provider_model_inventory, model_inventory_api_base, model_inventory_identity, - prevent_supplier_device_login, ) from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY from litellm.proxy.common_utils.encrypt_decrypt_utils import model_offering_identity_fingerprint @@ -274,7 +274,7 @@ class ModelOfferingsManager: if _unavailable_reason(offering, providers[offering.provider]) is None ) try: - with prevent_supplier_device_login(): + with prevent_device_login(): candidate: Final = ( self.template.snapshot_with_model_list(deployments, metadata_authoritative=True) if deployments != self.deployments diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index 5a4de6c1a31..8c459d4406a 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -11,7 +11,7 @@ from starlette.types import ASGIApp, Receive, Scope, Send import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm.llms.model_inventory import prevent_supplier_device_login +from litellm.llms.chatgpt.authenticator import prevent_device_login from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator, alias_target, caller_alias_maps from litellm.router import Router @@ -121,7 +121,7 @@ class OfferingRouterView(Router): def pin_snapshot(self) -> Generator[None]: token: Final = _PINNED_SNAPSHOT.set((id(self), self._offering_snapshot)) try: - with prevent_supplier_device_login(): + with prevent_device_login(): yield finally: _PINNED_SNAPSHOT.reset(token) diff --git a/litellm/types/proxy/model_metadata.py b/litellm/types/proxy/model_metadata.py deleted file mode 100644 index 92783573743..00000000000 --- a/litellm/types/proxy/model_metadata.py +++ /dev/null @@ -1,11 +0,0 @@ -from litellm.types.router import ( - ModelMetadata as GatewayModelMetadata, -) -from litellm.types.router import ( - ModelRequestDefaults, -) -from litellm.types.router import ( - resolve_model_metadata as resolve_gateway_model_metadata, -) - -__all__ = ("GatewayModelMetadata", "ModelRequestDefaults", "resolve_gateway_model_metadata") diff --git a/tests/unit/llms/chatgpt/test_model_info.py b/tests/unit/llms/chatgpt/test_model_info.py index f4c4dbc1454..a5177ba5350 100644 --- a/tests/unit/llms/chatgpt/test_model_info.py +++ b/tests/unit/llms/chatgpt/test_model_info.py @@ -11,7 +11,6 @@ from litellm.llms.chatgpt.common_utils import GetAccessTokenError from litellm.llms.chatgpt.model_info import ( _ChatGPTModel, chatgpt_model_inventory_identity, - get_chatgpt_model_info, get_chatgpt_model_inventory, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -163,14 +162,14 @@ async def test_account_catalog_preserves_client_policy_separately_from_model_lim async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: handler.client = client cache: Final = InMemoryCache() - result: Final = await get_chatgpt_model_info( - model="fixture-model", + result: Final = await get_chatgpt_model_inventory( api_base="https://account.test/backend-api/codex/", authenticator=Authenticator(), client=handler, cache=cache, ) - assert result == { + assert isinstance(result, SupplierModelInventory) + assert result.models["fixture-model"] == { "codex_context_window": card["context_window"], "codex_max_context_window": card["max_context_window"], "supports_parallel_function_calling": card["supports_parallel_tool_calls"], @@ -180,16 +179,14 @@ async def test_account_catalog_preserves_client_policy_separately_from_model_lim "supported_endpoints": ("/responses",), "supported_modalities": tuple(card["input_modalities"]), } - assert ( - await get_chatgpt_model_info( - model="not-selected", - api_base="https://account.test/backend-api/codex/", - authenticator=Authenticator(), - client=handler, - cache=cache, - ) - == {} + cached: Final = await get_chatgpt_model_inventory( + api_base="https://account.test/backend-api/codex/", + authenticator=Authenticator(), + client=handler, + cache=cache, ) + assert cached is result + assert cached.models.get("not-selected", {}) == {} async def test_switching_accounts_invalidates_catalog_without_exposing_credentials( @@ -212,15 +209,15 @@ async def test_switching_accounts_invalidates_catalog_without_exposing_credentia auth_file.write_text( json.dumps({"access_token": "local-token", "account_id": account, "expires_at": 10**30}) ) - result: Final = await get_chatgpt_model_info( - model="fixture", + result: Final = await get_chatgpt_model_inventory( api_base="https://account.test/backend-api/codex", authenticator=Authenticator(), client=handler, cache=cache, ) - assert result["codex_context_window"] == capacity - assert "local-token" not in json.dumps(dict(result)) + assert isinstance(result, SupplierModelInventory) + assert result.models["fixture"]["codex_context_window"] == capacity + assert "local-token" not in json.dumps(dict(result.models["fixture"])) async def test_discovery_without_authorization_never_starts_device_login( @@ -238,15 +235,13 @@ async def test_discovery_without_authorization_never_starts_device_login( await handler.client.aclose() async with httpx.AsyncClient(transport=httpx.MockTransport(reject)) as client: handler.client = client - assert ( - await get_chatgpt_model_info( - model="fixture", - authenticator=Authenticator(), - client=handler, - cache=InMemoryCache(), - ) - == {} + result: Final = await get_chatgpt_model_inventory( + authenticator=Authenticator(), + client=handler, + cache=InMemoryCache(), ) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == "authentication" assert not (tmp_path / "auth.json").exists() @@ -261,15 +256,13 @@ async def test_invalid_or_unavailable_catalog_preserves_fallback( await handler.client.aclose() async with httpx.AsyncClient(transport=httpx.MockTransport(lambda _: httpx.Response(status, json=body))) as client: handler.client = client - assert ( - await get_chatgpt_model_info( - model="fixture", - authenticator=Authenticator(), - client=handler, - cache=InMemoryCache(), - ) - == {} + result: Final = await get_chatgpt_model_inventory( + authenticator=Authenticator(), + client=handler, + cache=InMemoryCache(), ) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == ("http" if status == 503 else "malformed") @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_model_discovery.py b/tests/unit/proxy/test_model_discovery.py index c56ec401051..af0e63400e2 100644 --- a/tests/unit/proxy/test_model_discovery.py +++ b/tests/unit/proxy/test_model_discovery.py @@ -3,11 +3,11 @@ from typing import Final import pytest from litellm.proxy.model_discovery import discover_model_metadata, guaranteed_token_limit -from litellm.types.proxy.model_metadata import GatewayModelMetadata, resolve_gateway_model_metadata +from litellm.types.router import ModelMetadata, resolve_model_metadata def test_configured_overrides_preserve_false_empty_and_independent_limits(): - metadata: Final = resolve_gateway_model_metadata( + metadata: Final = resolve_model_metadata( { "context_window": 1200, "max_input_tokens": 1100, @@ -30,7 +30,7 @@ def test_configured_overrides_preserve_false_empty_and_independent_limits(): def test_mixed_deployments_advertise_only_guaranteed_metadata(): sources: Final = ( - GatewayModelMetadata( + ModelMetadata( context_window=1200, max_input_tokens=1100, max_output_tokens=200, @@ -40,7 +40,7 @@ def test_mixed_deployments_advertise_only_guaranteed_metadata(): default_reasoning_effort="high", supported_modalities=["text", "image"], ), - GatewayModelMetadata( + ModelMetadata( context_window=800, max_input_tokens=700, max_output_tokens=100, @@ -62,6 +62,6 @@ def test_mixed_deployments_advertise_only_guaranteed_metadata(): @pytest.mark.parametrize("value", [None, True, 0, -1, "1200", 1.5]) def test_unknown_or_invalid_context_never_fabricates_capacity(value): - metadata: Final = GatewayModelMetadata(context_window=value, max_input_tokens=1100, max_output_tokens=200) + metadata: Final = ModelMetadata(context_window=value, max_input_tokens=1100, max_output_tokens=200) assert discover_model_metadata((metadata,)) == {} - assert discover_model_metadata((GatewayModelMetadata(context_window=1200), metadata)) == {} + assert discover_model_metadata((ModelMetadata(context_window=1200), metadata)) == {} diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index bc37d19617c..d0b3b2623fd 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -21,7 +21,7 @@ from litellm.proxy.offering_router import ( OfferingSnapshotMiddleware, ) from litellm.router import Router -from litellm.types.proxy.model_offerings import OpenRouterProviderPolicy +from litellm.types.llms.openrouter import OpenRouterProviderPolicy from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes diff --git a/tests/unit/router_utils/test_model_request_defaults.py b/tests/unit/router_utils/test_model_request_defaults.py index 1b0a05448b4..5b0fb8468b0 100644 --- a/tests/unit/router_utils/test_model_request_defaults.py +++ b/tests/unit/router_utils/test_model_request_defaults.py @@ -5,7 +5,7 @@ from pydantic import ValidationError from litellm.proxy.model_discovery import discover_model_metadata from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults -from litellm.types.proxy.model_metadata import GatewayModelMetadata, ModelRequestDefaults +from litellm.types.router import ModelMetadata, ModelRequestDefaults _POLICY: Final = { "output_token_budget": 8192, @@ -20,7 +20,7 @@ def test_request_budget_follows_effective_reasoning_without_claiming_supplier_li metadata: Final = {"context_window": 262144, "request_defaults": _POLICY} request: Final = {"reasoning_effort": effort} assert model_request_defaults(metadata, {}, request, "completion") == {"max_tokens": expected} - assert "max_output_tokens" not in discover_model_metadata((GatewayModelMetadata.model_validate(metadata),)) + assert "max_output_tokens" not in discover_model_metadata((ModelMetadata.model_validate(metadata),)) assert request == {"reasoning_effort": effort} @@ -85,10 +85,10 @@ def test_invalid_request_budgets_are_rejected(value, field): def test_request_budget_maps_are_immutable_and_only_common_policy_is_advertised(): - metadata: Final = GatewayModelMetadata.model_validate({"request_defaults": _POLICY}) + metadata: Final = ModelMetadata.model_validate({"request_defaults": _POLICY}) assert discover_model_metadata((metadata, metadata)) == {"request_defaults": _POLICY} - assert discover_model_metadata((metadata, GatewayModelMetadata())) == {} - other: Final = GatewayModelMetadata(request_defaults=ModelRequestDefaults(output_token_budget=16384)) + assert discover_model_metadata((metadata, ModelMetadata())) == {} + other: Final = ModelMetadata(request_defaults=ModelRequestDefaults(output_token_budget=16384)) assert discover_model_metadata((metadata, other)) == {} defaults: Final = metadata.request_defaults assert defaults is not None diff --git a/tests/unit/types/proxy/test_model_offerings.py b/tests/unit/types/proxy/test_model_offerings.py index 019dc5fa093..9b791caaec4 100644 --- a/tests/unit/types/proxy/test_model_offerings.py +++ b/tests/unit/types/proxy/test_model_offerings.py @@ -3,7 +3,8 @@ from typing import Final import pytest from pydantic import ValidationError -from litellm.types.proxy.model_offerings import ModelOfferingsConfig, OpenRouterProviderPolicy +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.proxy.model_offerings import ModelOfferingsConfig @pytest.mark.parametrize( From d3ee59108701a7cb12702313cdcb3ffacdc1916a Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Tue, 13 Oct 2026 20:43:20 +0200 Subject: [PATCH 25/34] test(ci): assign model inventory regressions to the existing shard --- .github/workflows/test-unit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index e9236ee7787..bb7a913c1cc 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -366,6 +366,7 @@ jobs: test-path: >- tests/unit/llms/openai tests/unit/llms/meta + tests/unit/llms/test_model_inventory.py python-version: "3.12" workers: 4 reruns: 0 From 7fca6ea187b92471d38e8cfa96115f46db5264fc Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Tue, 13 Oct 2026 22:16:28 +0200 Subject: [PATCH 26/34] fix(proxy): compare typed model metadata echoes in their wire format --- .../model_management_endpoints.py | 4 +- .../test_model_management_endpoints.py | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 673ea829723..8b0d70c8b2a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1041,7 +1041,7 @@ def update_db_model( _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) merged_model_name: Final = updated_patch.model_name or db_model.model_name merged_litellm_params: Final = db_model.litellm_params.model_dump(exclude_none=True) - stored_model_info: Final = _dump_without_none(db_model.model_info) + stored_model_info: Final = db_model.model_info.model_dump(mode="json", exclude_none=True) echoed_pricing: Final = echoed_cost_map_pricing_fields(stored_model_info) merged_model_info: Final[dict[str, object]] = { k: v for k, v in stored_model_info.items() if k not in echoed_pricing @@ -1063,7 +1063,7 @@ def update_db_model( # update model info if updated_patch.model_info: - incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) + incoming_model_info: Final = updated_patch.model_info.model_dump(mode="json", exclude_none=True) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, _cost_map_entry(db_model, incoming_model_info), diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index beffe454b8d..306dce8e8c6 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4115,6 +4115,45 @@ class TestModelInfoCostMapEchoFilter: "cost-map metadata must not be persisted from an unchanged /model/info echo" ) + @pytest.mark.parametrize("case", ("echo", "edited", "stored")) + def test_typed_metadata_compares_wire_values_without_freezing_catalog_echoes( + self, monkeypatch: pytest.MonkeyPatch, case: str + ) -> None: + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + + catalog: Final = { + "mode": "chat", + "litellm_provider": "openai", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + "request_defaults": {"output_token_budget": 100, "output_token_budget_by_reasoning_effort": {"max": 180}}, + } + monkeypatch.setattr(litellm, "model_cost", {"openai/metadata-fixture": catalog}) + entry: Final = litellm.get_model_info("openai/metadata-fixture") + fields: Final = frozenset(catalog) - {"mode", "litellm_provider"} + stored: Final = {field: catalog[field] for field in fields} if case == "stored" else {} + edited: Final = {"supported_modalities": ["text"]} if case == "edited" else {} + deployment: Final = Deployment( + model_name="public-model", + litellm_params=LiteLLM_Params(model="openai/metadata-fixture"), + model_info=ModelInfo(id="private-deployment", **stored), + ) + result: Final = update_db_model( + db_model=deployment, + updated_patch=updateDeployment(model_info=ModelInfo(**{**entry, **edited, "access_groups": ["allowed"]})), + loaded_catalog=lambda: {"openai/metadata-fixture": catalog}, + ) + saved: Final = json.loads(result["model_info"]) + assert saved["access_groups"] == ["allowed"] + assert {field: value for field, value in saved.items() if field in fields} == ( + stored if case == "stored" else edited + ) + def test_an_edited_value_survives_the_echo_filter(self): import litellm From 84858032cc22a739c87fafcbdc5e1b0ab514a77e Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 15 Oct 2026 12:40:38 +0200 Subject: [PATCH 27/34] feat(openrouter): discover policy-scoped endpoint metadata and preserve effort values --- examples/model_offerings/README.md | 14 +- litellm/llms/anthropic/pass_through/utils.py | 2 + litellm/llms/model_inventory.py | 34 +- litellm/llms/openai_like/model_info.py | 42 +- .../llms/openrouter/chat/transformation.py | 4 - litellm/llms/openrouter/model_info.py | 206 ++++++++++ litellm/proxy/model_discovery.py | 62 +-- litellm/proxy/model_offerings.py | 41 +- litellm/proxy/offering_router.py | 5 +- litellm/proxy/utils.py | 4 +- litellm/types/llms/openrouter.py | 12 +- litellm/types/proxy/model_inventory.py | 13 +- litellm/types/router.py | 58 +++ .../test_reasoning_effort_fields.py | 8 + .../test_openrouter_chat_transformation.py | 6 +- tests/unit/llms/openrouter/test_model_info.py | 364 ++++++++++++++++++ tests/unit/proxy/test_model_discovery.py | 4 +- tests/unit/proxy/test_model_offerings.py | 244 +++++++++++- tests/unit/proxy/test_offering_router.py | 2 + tests/unit/test_internal_context.py | 1 + 20 files changed, 1028 insertions(+), 98 deletions(-) create mode 100644 litellm/llms/openrouter/model_info.py create mode 100644 tests/unit/llms/openrouter/test_model_info.py diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index 6d1f856af2d..62c54f42fd1 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -89,15 +89,23 @@ offerings: max_output_tokens: 4000 ``` -Append this entry to the external file's existing `offerings`, using its configured OpenRouter connection name. The example limits are synthetic. Verify endpoint-specific values before enabling it. `only` contains provider slugs, not URLs, and a single full slug pins one endpoint. Multiple explicit slugs intentionally allow any endpoint in that list; `allow_fallbacks` must remain false +Append this entry to the external file's existing `offerings`, using its configured OpenRouter connection name. The example limits are synthetic overrides. Omit them to discover bounds automatically. `only` contains provider slugs, not URLs. Base slugs include their variants, and variant slugs include their regions, following OpenRouter's matching contract. Service-tier endpoints require an explicit tier slug. Multiple slugs intentionally allow any matching endpoint; `allow_fallbacks` must remain false -The gateway merges its policy into harmless caller `extra_body` fields and verifies it again at deployment dispatch. Caller `provider`, `models`, `route`, fallback controls, parameter-dropping/allowlist controls, nested body overrides and mock responses are rejected for policy-controlled offerings, even through aliases. Caller deployment selectors are rejected throughout external mode. Policy-controlled requests disable LiteLLM model-group fallbacks as well as OpenRouter provider fallback. Unavailable endpoints fail instead of serving a different offering. Retries may still contact the same pinned endpoint +The gateway merges its policy into harmless caller `extra_body` fields and verifies it again at deployment dispatch. Caller `provider`, `models`, `route`, `service_tier`, fallback controls, parameter-dropping/allowlist controls, nested body overrides and mock responses are rejected for policy-controlled offerings, even through aliases. Tier overrides could change the eligible endpoint set and invalidate its advertised guarantees; they are rejected rather than silently reduced or rerouted. Unpinned tier controls remain unchanged. Caller deployment selectors are rejected throughout external mode. Policy-controlled requests disable LiteLLM model-group fallbacks as well as OpenRouter provider fallback. Unavailable endpoints fail instead of serving a different offering. Retries may still contact the same pinned endpoint Policy-controlled requests disable response-cache reads and writes, overriding caller cache settings. Existing response-cache keys do not reliably separate provider endpoints; cached output from another endpoint must not satisfy a pinned request Chat Completions, native Responses and translated Anthropic Messages preserve the policy, including streaming. Other APIs are rejected for policy-controlled offerings. Batch execution remains unavailable throughout external mode. Offerings without policy retain existing provider-routing behavior, apart from the deployment-selector restriction -AUTO still checks model membership, but a pinned endpoint does not inherit aggregate supplier or bundled model metadata. Only explicit `model_info` is advertised, with unknown limits omitted. Endpoint inventory polling is not added: verify and maintain those overrides separately. Policy edits change that offering's private deployment ID and publish atomically; requests already streaming retain their previous snapshot +AUTO still checks model membership. Pinned AUTO offerings additionally discover only their selected models' endpoint inventories, using the configured connection, bounded native HTTP client and credential-scoped cache. Aggregate model limits and bundled metadata never substitute for pinned endpoint limits. Context, independent input and output bounds are the conservative minimum across all eligible endpoints, and a field unknown on any eligible endpoint is omitted. Input capacity is not inferred by subtracting output from context + +Reasoning choices and defaults come from OpenRouter's model-level reasoning contract and are advertised only when every eligible endpoint declares the effort control. An endpoint's more restrictive declared choices narrow that set. This describes OpenRouter's accepted gateway options, not an independently discovered backend-native effort enum. An explicit empty choice list remains empty. Explicit null expands only to OpenRouter's documented gateway efforts, while omission does not declare choices. Mandatory reasoning excludes the disable value, and a disabled-by-default model does not claim an active default effort. Boolean reasoning support alone never supplies choices. Endpoint capability guarantees are combined conservatively, without using status, uptime or performance to change routing or availability + +Every explicit `model_info` field remains authoritative. Existing handwritten context/output limits continue overriding discovery until the operator removes them. MANUAL offerings remain entirely handwritten. Requests without an effort leave the supplier default unchanged; explicit gateway effort values, including `max` and `xhigh`, are forwarded without backend-flag approximation on chat and translated Messages paths + +Successful empty endpoint inventories clear derived endpoint metadata. Failed refreshes retain last-known endpoint metadata only for the same connection identity; an initial failure leaves unknown fields omitted, never replaced with another supplier's limits. Endpoint discovery does not disable an otherwise selected model or broaden its policy. Metadata refreshes publish atomically, and requests already running retain their previous snapshot. Policy edits still change the affected offering's private deployment ID + +The endpoint and reasoning provenance follows OpenRouter's [endpoint API](https://openrouter.ai/docs/api/api-reference/endpoints/list-all-endpoints-for-a-model), [per-model reasoning options](https://openrouter.ai/docs/guides/best-practices/reasoning-tokens#discovering-per-model-reasoning-options) and [provider slug matching](https://openrouter.ai/docs/guides/routing/provider-selection#base-slug-matching), checked October 15, 2026 Native provider batch creation and uploads with purpose `batch` are rejected in external offering mode. Native batch records can select their own supplier models, which would bypass the configured offering catalog. Existing proxy configurations without external offerings retain native batch support. diff --git a/litellm/llms/anthropic/pass_through/utils.py b/litellm/llms/anthropic/pass_through/utils.py index 95baecbfbfe..e1f7b6b9ca6 100644 --- a/litellm/llms/anthropic/pass_through/utils.py +++ b/litellm/llms/anthropic/pass_through/utils.py @@ -82,6 +82,8 @@ def normalize_reasoning_effort_value( handled where the thinking block is built. A deployment accepting no tier at all keeps the chain's floor, which is what every deployment degraded to before there was anything to ask. """ + if custom_llm_provider == "openrouter" or custom_llm_provider is None and model.startswith("openrouter/"): + return effort chain: Final = _EFFORT_DEGRADATION_CHAIN.get(effort) if chain is None: return effort diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py index 76fd286c525..55473295b6a 100644 --- a/litellm/llms/model_inventory.py +++ b/litellm/llms/model_inventory.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # native HTTP/cache protocol boundary from pydantic import TypeAdapter @@ -15,7 +15,14 @@ from litellm.llms.chatgpt.model_info import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_DISCOVERY_PROVIDERS, get_openai_compatible_model_inventory -from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.llms.openrouter.model_info import get_openrouter_endpoint_inventory, openrouter_endpoint_metadata +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelInventory, +) from litellm.types.router import LiteLLM_Params _EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) @@ -44,6 +51,8 @@ async def get_provider_model_inventory( client: AsyncHTTPHandler, cache: InMemoryCache, force_refresh: bool = False, + endpoint_models: tuple[str, ...] = (), + previous: SupplierModelInventory | None = None, ) -> SupplierModelInventory | SupplierInventoryUnavailable: if provider == "chatgpt": return await get_chatgpt_model_inventory( @@ -55,7 +64,7 @@ async def get_provider_model_inventory( resolved_base: Final = model_inventory_api_base(provider, api_base) if resolved_base is None: return SupplierInventoryUnavailable("malformed") - return await get_openai_compatible_model_inventory( + inventory: Final = await get_openai_compatible_model_inventory( provider=provider, api_base=resolved_base, headers=headers, @@ -63,6 +72,25 @@ async def get_provider_model_inventory( cache=cache, force_refresh=force_refresh, ) + if provider != "openrouter" or not isinstance(inventory, SupplierModelInventory) or not endpoint_models: + return inventory + return await get_openrouter_endpoint_inventory( + inventory=inventory, + models=endpoint_models, + previous=previous, + api_base=resolved_base, + headers=headers, + client=cast(ModelInventoryHTTPClient, client), # cast-ok: native handler implements bounded inventory GET + cache=cast(ModelInventoryCache, cache), # cast-ok: native in-memory cache implements inventory protocol + force_refresh=force_refresh, + ) + + +def get_inventory_model_metadata( + inventory: SupplierModelInventory, model: str, policy: OpenRouterProviderPolicy | None = None +) -> Mapping[str, object]: + metadata: Final = inventory.models.get(model, _EMPTY_METADATA) + return openrouter_endpoint_metadata(metadata, inventory.endpoints.get(model, ()), policy) if policy else metadata async def get_deployment_model_metadata( diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 724775b671a..14a8abf2ac5 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -11,6 +11,7 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.core_helpers import coerce_token_limit from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openrouter import OPENROUTER_REASONING_EFFORTS from litellm.types.proxy.model_inventory import ( ModelInventoryCache, ModelInventoryHTTPClient, @@ -28,6 +29,15 @@ MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset( _EMPTY_LIMITS: Final[Mapping[str, object]] = MappingProxyType({}) +def model_inventory_cache_key(provider: str, url: str, headers: Mapping[str, str]) -> str: + return ( + "upstream_model_info:" + + hashlib.sha256( + json.dumps((provider, url, sorted((key.lower(), value) for key, value in headers.items()))).encode() + ).hexdigest() + ) + + def _positive_limit(value: object) -> int | None: if value is None or type(value) is int and value == 0: return None @@ -66,6 +76,21 @@ class _Reasoning(LiteLLMBaseModel): supported_efforts: Sequence[str] | None = None default_effort: str | None = None + default_enabled: bool | None = None + mandatory: bool | None = None + + def effort_levels(self) -> Sequence[str] | None: + """Explicit null accepts the gateway enum; omission does not declare an effort selector.""" + levels: Final = ( + OPENROUTER_REASONING_EFFORTS + if self.supported_efforts is None and "supported_efforts" in self.model_fields_set + else self.supported_efforts + ) + return ( + tuple(level for level in levels if level != "none" or self.mandatory is not True) + if levels is not None + else None + ) class _ReasoningOption(LiteLLMBaseModel): @@ -126,7 +151,7 @@ class _ModelCard(ModelMetadata): self.reasoning_effort_levels if self.reasoning_effort_levels is not None else ( - self.reasoning.supported_efforts + self.reasoning.effort_levels() if provider == "openrouter" and self.reasoning is not None else next((option.values for option in self.reasoning_options if option.type == "effort"), None) if provider == "vercel_ai_gateway" @@ -134,7 +159,13 @@ class _ModelCard(ModelMetadata): ) ), default_reasoning_effort=self.default_reasoning_effort - or (self.reasoning.default_effort if provider == "openrouter" and self.reasoning is not None else None), + or ( + self.reasoning.default_effort + if provider == "openrouter" + and self.reasoning is not None + and (self.reasoning.default_enabled is not False or self.reasoning.default_effort == "none") + else None + ), supported_endpoints=self.supported_endpoints, request_defaults=self.request_defaults, supported_modalities=self.supported_modalities @@ -208,10 +239,7 @@ async def get_openai_compatible_model_inventory( inventory_cache: Final = cast(ModelInventoryCache, cache) # cast-ok: native cache implements this protocol inventory_client: Final = cast(ModelInventoryHTTPClient, client) # cast-ok: native HTTP handler implements get url: Final = add_path_to_api_base(api_base, "/v1/models") - cache_key: Final = ( - "upstream_model_info:" - + hashlib.sha256(json.dumps((provider, url, sorted(headers.items()))).encode()).hexdigest() - ) + cache_key: Final = model_inventory_cache_key(provider, url, headers) cached: Final[object] = inventory_cache.get_cache(cache_key) if not force_refresh and isinstance(cached, (SupplierModelInventory, SupplierInventoryUnavailable)): return cached @@ -249,7 +277,7 @@ async def get_openai_compatible_model_inventory( return transport inventory: Final = SupplierModelInventory( - MappingProxyType({card.id: card.token_limits(provider) for card in models.data}) + MappingProxyType({card.id: card.token_limits(provider) for card in models.data}), credential_scope=cache_key ) inventory_cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) return inventory diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 7cdd3978356..222922ac107 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -61,10 +61,6 @@ class OpenrouterConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: - # OpenRouter expects "xhigh" instead of "max" for reasoning_effort. - if non_default_params.get("reasoning_effort") == "max": - non_default_params = {**non_default_params, "reasoning_effort": "xhigh"} - mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) # OpenRouter-only parameters diff --git a/litellm/llms/openrouter/model_info.py b/litellm/llms/openrouter/model_info.py new file mode 100644 index 00000000000..53879006f1c --- /dev/null +++ b/litellm/llms/openrouter/model_info.py @@ -0,0 +1,206 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final +from urllib.parse import quote + +import httpx +from pydantic import ConfigDict, ValidationError, ValidationInfo, field_validator + +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS, model_inventory_cache_key +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelEndpoint, + SupplierModelInventory, +) +from litellm.types.router import ModelMetadata, common_model_metadata +from litellm.utils import add_path_to_api_base + + +class _Endpoint(ModelMetadata): + tag: str + context_length: int | None = None + max_prompt_tokens: int | None = None + max_completion_tokens: int | None = None + supported_parameters: tuple[str, ...] | None = None + + @field_validator("context_length", "max_prompt_tokens", "max_completion_tokens", mode="before") + @classmethod + def known_limit(cls, value: object) -> object: + if value is None or type(value) is int and value >= 0: + return value + raise ValueError("Endpoint limits must be nonnegative integers or null") + + @field_validator("*", mode="before") + @classmethod + def strict_metadata(cls, value: object, info: ValidationInfo) -> object: + return ModelMetadata.validate_supplier_field(info.field_name, value) + + def normalized(self) -> SupplierModelEndpoint: + parameters: Final = self.supported_parameters + return SupplierModelEndpoint( + self.tag, + ModelMetadata.model_validate( + { + **self.model_dump(exclude_none=True), + "context_window": self.context_length, + "max_input_tokens": self.max_prompt_tokens, + "max_output_tokens": self.max_completion_tokens, + "supports_function_calling": self.supports_function_calling + if self.supports_function_calling is not None + else "tools" in parameters + if parameters is not None + else None, + "supports_parallel_function_calling": self.supports_parallel_function_calling + if self.supports_parallel_function_calling is not None + else True + if parameters is not None and "parallel_tool_calls" in parameters + else None, + "supports_reasoning": self.supports_reasoning + if self.supports_reasoning is not None + else bool({"reasoning", "reasoning_effort"}.intersection(parameters)) + if parameters is not None + else None, + } + ), + parameters, + ) + + +class _EndpointData(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + id: str + endpoints: tuple[_Endpoint, ...] + + +class _EndpointResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + data: _EndpointData + error: object | None = None + + +async def _fetch_endpoints( + model: str, + *, + api_base: str, + headers: Mapping[str, str], + client: ModelInventoryHTTPClient, + cache: ModelInventoryCache, + force_refresh: bool, +) -> tuple[SupplierModelEndpoint, ...] | SupplierInventoryUnavailable: + if len(model.split("/")) != 2 or any(part in ("", ".", "..") for part in model.split("/")): + return SupplierInventoryUnavailable("malformed") + url: Final = add_path_to_api_base(api_base, f"/v1/models/{quote(model, safe='/')}/endpoints") + cache_key: Final = model_inventory_cache_key("openrouter-endpoints", url, headers) + cached: Final[object] = cache.get_cache(cache_key) + if not force_refresh and isinstance(cached, (_EndpointData, SupplierInventoryUnavailable)): + return ( + tuple(endpoint.normalized() for endpoint in cached.endpoints) + if isinstance(cached, _EndpointData) + else cached + ) + try: + response: Final = await client.get( + url=url, + headers=dict(headers), + timeout=httpx.Timeout(5.0), + follow_redirects=False, + max_response_bytes=2 * 1024 * 1024, + ) + response.raise_for_status() + parsed: Final = _EndpointResponse.model_validate_json(response.content) + data: Final = parsed.data + if ( + parsed.error is not None + or data.id != model + or any(not endpoint.tag.strip() or endpoint.tag != endpoint.tag.strip() for endpoint in data.endpoints) + ): + raise ValueError("Endpoint inventory must identify the requested model and provider slugs") + except httpx.HTTPStatusError: + unavailable: Final = SupplierInventoryUnavailable("http") + cache.set_cache(cache_key, unavailable, ttl=60) + return unavailable + except (ValidationError, ValueError): + malformed: Final = SupplierInventoryUnavailable("malformed") + cache.set_cache(cache_key, malformed, ttl=60) + return malformed + except Exception: # noqa: BLE001 # optional metadata must not interrupt the last-valid serving generation + transport: Final = SupplierInventoryUnavailable("transport") + cache.set_cache(cache_key, transport, ttl=60) + return transport + cache.set_cache(cache_key, data, ttl=MODEL_INFO_REFRESH_SECONDS) + return tuple(endpoint.normalized() for endpoint in data.endpoints) + + +async def get_openrouter_endpoint_inventory( + *, + inventory: SupplierModelInventory, + models: tuple[str, ...], + previous: SupplierModelInventory | None, + api_base: str, + headers: Mapping[str, str], + client: ModelInventoryHTTPClient, + cache: ModelInventoryCache, + force_refresh: bool, +) -> SupplierModelInventory: + selected: Final = tuple(dict.fromkeys(model for model in models if model in inventory.models)) + retained: Final[Mapping[str, tuple[SupplierModelEndpoint, ...]]] = ( + previous.endpoints + if previous is not None + and inventory.credential_scope is not None + and previous.credential_scope == inventory.credential_scope + else MappingProxyType({}) + ) + results: Final = tuple( + [ + await _fetch_endpoints( + model, api_base=api_base, headers=headers, client=client, cache=cache, force_refresh=force_refresh + ) + for model in selected + ] + ) + return SupplierModelInventory( + inventory.models, + inventory.credential_scope, + MappingProxyType( + { + model: result if isinstance(result, tuple) else retained.get(model, ()) + for model, result in zip(selected, results) + } + ), + ) + + +def openrouter_endpoint_metadata( + model_info: Mapping[str, object], endpoints: tuple[SupplierModelEndpoint, ...], policy: OpenRouterProviderPolicy +) -> Mapping[str, object]: + eligible: Final = tuple(endpoint for endpoint in endpoints if policy.allows_endpoint(endpoint.tag)) + model: Final = ModelMetadata.model_validate(model_info) + sources: Final = tuple(_endpoint_metadata(model, endpoint) for endpoint in eligible) + return MappingProxyType(common_model_metadata(sources).model_dump(mode="json", exclude_none=True)) + + +def _endpoint_metadata(model: ModelMetadata, endpoint: SupplierModelEndpoint) -> ModelMetadata: + supports_effort: Final = ( + endpoint.supported_parameters is not None and "reasoning_effort" in endpoint.supported_parameters + ) + endpoint_levels: Final = endpoint.metadata.reasoning_effort_levels + levels: Final = ( + model.reasoning_effort_levels + if endpoint_levels is None + else endpoint_levels + if model.reasoning_effort_levels is None + else tuple(level for level in model.reasoning_effort_levels if level in endpoint_levels) + ) + return ModelMetadata.model_validate( + { + **endpoint.metadata.model_dump(exclude_none=True), + "reasoning_effort_levels": levels if supports_effort else None, + "default_reasoning_effort": (endpoint.metadata.default_reasoning_effort or model.default_reasoning_effort) + if supports_effort + else None, + } + ) diff --git a/litellm/proxy/model_discovery.py b/litellm/proxy/model_discovery.py index 2406e3f5175..0eb5a3bdc2a 100644 --- a/litellm/proxy/model_discovery.py +++ b/litellm/proxy/model_discovery.py @@ -1,66 +1,16 @@ -from collections.abc import Sequence -from typing import Final, Literal +from typing import Final from pydantic import TypeAdapter from litellm.types.proxy.model_listing import ModelDiscoveryInfo -from litellm.types.router import ModelMetadata +from litellm.types.router import ModelMetadata, common_model_metadata _ADAPTER: Final = TypeAdapter(ModelDiscoveryInfo) -def guaranteed_token_limit( - sources: tuple[ModelMetadata, ...], field: Literal["context_window", "max_input_tokens", "max_output_tokens"] -) -> int | None: - values: Final = tuple( - source.context_window - if field == "context_window" - else source.max_input_tokens - if field == "max_input_tokens" - else source.max_output_tokens - for source in sources - ) - if not values or any(value is None for value in values): - return None - return min(value for value in values if value is not None) - - -def _common_boolean(values: tuple[bool | None, ...]) -> bool | None: - return all(values) if values and all(value is not None for value in values) else None - - -def _common_list(values: tuple[Sequence[str] | None, ...]) -> Sequence[str] | None: - if not values or any(value is None for value in values): - return None - first: Final = values[0] - if first is None: - return None - return tuple(item for item in first if all(value is not None and item in value for value in values)) - - def discover_model_metadata(sources: tuple[ModelMetadata, ...]) -> ModelDiscoveryInfo: - efforts: Final = _common_list(tuple(source.reasoning_effort_levels for source in sources)) - defaults: Final = tuple(source.default_reasoning_effort for source in sources) - default: Final = defaults[0] if defaults and all(value == defaults[0] for value in defaults) else None - valid_default: Final = default if efforts is None or default in efforts else None - request_defaults: Final = tuple(source.request_defaults for source in sources) - common_request_defaults: Final = ( - request_defaults[0] - if request_defaults and all(value == request_defaults[0] for value in request_defaults) - else None + return _ADAPTER.validate_python( + common_model_metadata(sources).model_dump( + mode="json", exclude_none=True, exclude={"max_input_tokens", "max_output_tokens"} + ) ) - metadata: Final = ModelMetadata( - context_window=guaranteed_token_limit(sources, "context_window"), - supports_function_calling=_common_boolean(tuple(source.supports_function_calling for source in sources)), - supports_parallel_function_calling=_common_boolean( - tuple(source.supports_parallel_function_calling for source in sources) - ), - supports_reasoning=_common_boolean(tuple(source.supports_reasoning for source in sources)), - reasoning_effort_levels=efforts, - default_reasoning_effort=valid_default, - supported_endpoints=_common_list(tuple(source.supported_endpoints for source in sources)), - supported_modalities=_common_list(tuple(source.supported_modalities for source in sources)), - supported_output_modalities=_common_list(tuple(source.supported_output_modalities for source in sources)), - request_defaults=common_request_defaults, - ) - return _ADAPTER.validate_python(metadata.model_dump(mode="json", exclude_none=True)) diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py index 1ff3d904f98..27edc86782a 100644 --- a/litellm/proxy/model_offerings.py +++ b/litellm/proxy/model_offerings.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.llms.chatgpt.authenticator import prevent_device_login from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.model_inventory import ( + get_inventory_model_metadata, get_provider_model_inventory, model_inventory_api_base, model_inventory_identity, @@ -159,12 +160,27 @@ def _deployment_id(offering: ModelOffering, connection: _ResolvedConnection) -> ) +def _endpoint_models(config: ModelOfferingsConfig | None, provider: str) -> tuple[str, ...]: + if config is None: + return () + return tuple( + dict.fromkeys( + offering.upstream_model + for offering in config.offerings + if offering.provider == provider + and offering.enabled + and offering.source == "auto" + and offering.provider_policy is not None + ) + ) + + def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, object]: connection: Final = state.connection reason: Final = _unavailable_reason(offering, state) supplier_info: Final[Mapping[str, object]] = ( - state.inventory.models.get(offering.upstream_model, {}) - if offering.source == "auto" and state.inventory is not None and offering.provider_policy is None + get_inventory_model_metadata(state.inventory, offering.upstream_model, offering.provider_policy) + if offering.source == "auto" and state.inventory is not None else {} ) catalog: Final = GetModelCostMap.loaded_model_cost_map() @@ -238,7 +254,7 @@ class ModelOfferingsManager: if isinstance(callback, EncryptedContentAffinityCheck) and callback.router is template: callback.router = self.router - async def _fetch_provider(self, state: _ProviderState) -> _ProviderState: + async def _fetch_provider(self, state: _ProviderState, endpoint_models: tuple[str, ...]) -> _ProviderState: connection: Final = state.connection try: fetched: Final = await get_provider_model_inventory( @@ -248,6 +264,8 @@ class ModelOfferingsManager: client=self.client or get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelInventory), cache=self.cache, force_refresh=True, + endpoint_models=endpoint_models, + previous=state.inventory, ) except Exception: # noqa: BLE001 # supplier failures cannot replace a successful inventory with absence return state @@ -258,13 +276,17 @@ class ModelOfferingsManager: ) return _provider_after_fetch(state, fetched) - async def _bounded_fetch_provider(self, state: _ProviderState) -> _ProviderState: + async def _bounded_fetch_provider(self, state: _ProviderState, endpoint_models: tuple[str, ...]) -> _ProviderState: async with self.inventory_semaphore: - return await self._fetch_provider(state) + return await self._fetch_provider(state, endpoint_models) - async def _refresh_providers(self, providers: Mapping[str, _ProviderState]) -> Mapping[str, _ProviderState]: + async def _refresh_providers( + self, providers: Mapping[str, _ProviderState], config: ModelOfferingsConfig + ) -> Mapping[str, _ProviderState]: names: Final = tuple(providers) - results: Final = await asyncio.gather(*(self._bounded_fetch_provider(providers[name]) for name in names)) + results: Final = await asyncio.gather( + *(self._bounded_fetch_provider(providers[name], _endpoint_models(config, name)) for name in names) + ) return MappingProxyType(dict(zip(names, results))) def _apply(self, loaded: _LoadedConfig, providers: Mapping[str, _ProviderState]) -> bool: @@ -361,9 +383,12 @@ class ModelOfferingsManager: or force_inventory or connection_changed or interval_changed + or any( + _endpoint_models(loaded.config, name) != _endpoint_models(self.config, name) for name in providers + ) or now >= self.next_inventory_refresh ) - refreshed: Final = await self._refresh_providers(providers) if refresh_due else providers + refreshed: Final = await self._refresh_providers(providers, loaded.config) if refresh_due else providers if not self._apply(loaded, refreshed): return False if refresh_due: diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index 8c459d4406a..eb0e41c688b 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -42,6 +42,7 @@ _ROUTING_OVERRIDES: Final = ( "provider", "models", "route", + "service_tier", "fallbacks", "default_fallbacks", "context_window_fallbacks", @@ -323,11 +324,11 @@ class OfferingAccessGuard(CustomLogger): call_type is None or call_type.value not in _POLICY_CALLS or any(kwargs.get(name) is not None for name in _PARAMETER_CONTROLS) - or any(kwargs.get(name) is not None for name in ("provider", "models", "route")) + or any(kwargs.get(name) is not None for name in ("provider", "models", "route", "service_tier")) or not isinstance(extra_body, Mapping) or extra_body.get("provider") != policy.request_body() or any(name in extra_body for name in _PARAMETER_CONTROLS) - or any(name in extra_body for name in ("model", "models", "route", "extra_body")) + or any(name in extra_body for name in ("model", "models", "route", "extra_body", "service_tier")) ): raise litellm.BadRequestError( message="The request does not preserve the configured offering provider policy", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 44f4d43797a..066c5d4e506 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -81,11 +81,11 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, with_litellm_call_id, ) -from litellm.proxy.model_discovery import discover_model_metadata, guaranteed_token_limit +from litellm.proxy.model_discovery import discover_model_metadata from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.router import resolve_model_metadata +from litellm.types.router import guaranteed_token_limit, resolve_model_metadata from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage try: diff --git a/litellm/types/llms/openrouter.py b/litellm/types/llms/openrouter.py index 1f66dc3de17..be7f7ec7640 100644 --- a/litellm/types/llms/openrouter.py +++ b/litellm/types/llms/openrouter.py @@ -1,12 +1,14 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Annotated, Literal +from typing import Annotated, Final, Literal from pydantic import ConfigDict, Field, field_validator from typing_extensions import TypedDict from litellm.types.llms.base import LiteLLMBaseModel +OPENROUTER_REASONING_EFFORTS: Final = ("max", "xhigh", "high", "medium", "low", "minimal", "none") + class OpenRouterErrorMessage(TypedDict): message: str @@ -40,3 +42,11 @@ class OpenRouterProviderPolicy(LiteLLMBaseModel): def request_body(self) -> Mapping[str, object]: return MappingProxyType({"only": list(self.only), "allow_fallbacks": self.allow_fallbacks}) + + def allows_endpoint(self, tag: str) -> bool: + return any( + tag == slug + or tag.startswith(slug + "/") + and not {"fast", "flex"}.intersection(tag[len(slug) + 1 :].split("/")) + for slug in self.only + ) diff --git a/litellm/types/proxy/model_inventory.py b/litellm/types/proxy/model_inventory.py index 17f41454f1d..5511ffeecad 100644 --- a/litellm/types/proxy/model_inventory.py +++ b/litellm/types/proxy/model_inventory.py @@ -1,14 +1,25 @@ from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field +from types import MappingProxyType from typing import Literal, Protocol import httpx +from litellm.types.router import ModelMetadata + + +@dataclass(frozen=True, slots=True) +class SupplierModelEndpoint: + tag: str + metadata: ModelMetadata + supported_parameters: tuple[str, ...] | None + @dataclass(frozen=True, slots=True) class SupplierModelInventory: models: Mapping[str, Mapping[str, object]] credential_scope: str | None = None + endpoints: Mapping[str, tuple[SupplierModelEndpoint, ...]] = field(default_factory=lambda: MappingProxyType({})) @dataclass(frozen=True, slots=True) diff --git a/litellm/types/router.py b/litellm/types/router.py index ba403a060c0..f75b9c2e53a 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -374,6 +374,64 @@ class ModelMetadata(LiteLLMBaseModel): return value if levels is None or isinstance(levels, (list, tuple)) and value in levels else None +def guaranteed_token_limit( + sources: tuple[ModelMetadata, ...], field: Literal["context_window", "max_input_tokens", "max_output_tokens"] +) -> int | None: + values: Final = tuple( + source.context_window + if field == "context_window" + else source.max_input_tokens + if field == "max_input_tokens" + else source.max_output_tokens + for source in sources + ) + if not values or any(value is None for value in values): + return None + return min(value for value in values if value is not None) + + +def _common_metadata_list(values: tuple[Sequence[str] | None, ...]) -> Sequence[str] | None: + if not values or any(value is None for value in values): + return None + first: Final = values[0] + if first is None: + return None + return tuple(item for item in first if all(value is not None and item in value for value in values)) + + +def _common_metadata_boolean(values: tuple[bool | None, ...]) -> bool | None: + return all(values) if values and all(value is not None for value in values) else None + + +def common_model_metadata(sources: tuple[ModelMetadata, ...]) -> ModelMetadata: + efforts: Final = _common_metadata_list(tuple(source.reasoning_effort_levels for source in sources)) + defaults: Final = tuple(source.default_reasoning_effort for source in sources) + default: Final = defaults[0] if defaults and all(value == defaults[0] for value in defaults) else None + request_defaults: Final = tuple(source.request_defaults for source in sources) + return ModelMetadata( + context_window=guaranteed_token_limit(sources, "context_window"), + max_input_tokens=guaranteed_token_limit(sources, "max_input_tokens"), + max_output_tokens=guaranteed_token_limit(sources, "max_output_tokens"), + supports_function_calling=_common_metadata_boolean( + tuple(source.supports_function_calling for source in sources) + ), + supports_parallel_function_calling=_common_metadata_boolean( + tuple(source.supports_parallel_function_calling for source in sources) + ), + supports_reasoning=_common_metadata_boolean(tuple(source.supports_reasoning for source in sources)), + reasoning_effort_levels=efforts, + default_reasoning_effort=default if efforts is None or default in efforts else None, + supported_endpoints=_common_metadata_list(tuple(source.supported_endpoints for source in sources)), + supported_modalities=_common_metadata_list(tuple(source.supported_modalities for source in sources)), + supported_output_modalities=_common_metadata_list( + tuple(source.supported_output_modalities for source in sources) + ), + request_defaults=request_defaults[0] + if request_defaults and all(value == request_defaults[0] for value in request_defaults) + else None, + ) + + def resolve_model_metadata(catalog: Mapping[str, object], configured: Mapping[str, object]) -> ModelMetadata: values: Final = { field: configured[field] if configured.get(field) is not None else catalog.get(field) diff --git a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py index e9450d025a6..35f891c8a97 100644 --- a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py @@ -23,6 +23,14 @@ from litellm.router_utils.reasoning_effort_capability import ( from litellm.utils import get_model_info +@pytest.mark.parametrize("effort", ("max", "xhigh", "high", "minimal", "none", "future-effort")) +@pytest.mark.parametrize(("model", "provider"), (("org/off-map", "openrouter"), ("openrouter/org/off-map", None))) +def test_openrouter_gateway_efforts_are_never_degraded_from_backend_flags( + effort: str, model: str, provider: str | None +) -> None: + assert normalize_reasoning_effort_value(effort, model, provider) == effort + + def _load_model_registry() -> Dict[str, Any]: """Load the root model_prices_and_context_window.json.""" json_path = os.path.join( diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 59355dfabcb..1d3043600a5 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -562,9 +562,9 @@ def test_openrouter_non_reasoning_models_support_reasoning_params(): assert "thinking" in supported_params -def test_openrouter_reasoning_effort_max_maps_to_xhigh(): +def test_openrouter_reasoning_effort_max_is_preserved(): """ - OpenRouter expects 'xhigh' instead of 'max' for reasoning_effort. + Preserve the exact gateway effort, which need not be the backend's alias. """ config = OpenrouterConfig() @@ -575,7 +575,7 @@ def test_openrouter_reasoning_effort_max_maps_to_xhigh(): drop_params=False, ) - assert result["reasoning_effort"] == "xhigh" + assert result["reasoning_effort"] == "max" def test_openrouter_reasoning_effort_max_does_not_mutate_caller_dict(): diff --git a/tests/unit/llms/openrouter/test_model_info.py b/tests/unit/llms/openrouter/test_model_info.py new file mode 100644 index 00000000000..c829f2dcfe7 --- /dev/null +++ b/tests/unit/llms/openrouter/test_model_info.py @@ -0,0 +1,364 @@ +from collections.abc import Mapping +from typing import Final +from unittest.mock import Mock + +import httpx +import pytest + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.model_inventory import get_inventory_model_metadata, get_provider_model_inventory +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.types.llms.openrouter import OPENROUTER_REASONING_EFFORTS, OpenRouterProviderPolicy +from litellm.types.proxy.model_inventory import SupplierModelInventory + + +def _endpoint(tag: str = "supplier/fp8", **fields: object) -> Mapping[str, object]: + return { + "tag": tag, + "context_length": 1200, + "max_prompt_tokens": None, + "max_completion_tokens": 300, + "supported_parameters": ["tools", "reasoning", "reasoning_effort"], + **fields, + } + + +async def _inventory( + handler: AsyncHTTPHandler, + cache: InMemoryCache, + *, + only: tuple[str, ...] = ("supplier/fp8",), + key: str = "fixture-key", + base: str = "https://supplier.test/custom/v1", + refresh: bool = False, + previous: SupplierModelInventory | None = None, + model: str = "org/model", +) -> tuple[SupplierModelInventory, Mapping[str, object]]: + inventory: Final = await get_provider_model_inventory( + provider="openrouter", + api_base=base, + headers={"authorization": key}, + client=handler, + cache=cache, + endpoint_models=(model, model), + force_refresh=refresh, + previous=previous, + ) + assert isinstance(inventory, SupplierModelInventory) + return inventory, get_inventory_model_metadata( + inventory, model, OpenRouterProviderPolicy(provider="openrouter", only=only) + ) + + +def _catalog(reasoning: Mapping[str, object] | None = None) -> Mapping[str, object]: + return { + "data": [ + { + "id": "org/model", + "context_length": 9000, + "top_provider": {"context_length": 8000, "max_completion_tokens": 7000}, + "supported_parameters": ["tools", "reasoning_effort"], + "reasoning": {"supported_efforts": ["max", "high", "low"], "default_effort": "high"} + if reasoning is None + else reasoning, + }, + {"id": "org/second"}, + ] + } + + +@pytest.mark.parametrize( + ("endpoints", "only", "context", "output", "input_limit", "efforts", "default", "reasoning"), + ( + ( + [_endpoint(), _endpoint("other", context_length=9000)], + ("supplier/fp8",), + 1200, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(), _endpoint("supplier/fp8/eu", context_length=900, max_completion_tokens=200)], + ("supplier",), + 900, + 200, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(max_prompt_tokens=500), _endpoint("second", max_prompt_tokens=400)], + ("supplier", "second"), + 1200, + 300, + 400, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(), _endpoint("second", supported_parameters=["reasoning"])], + ("supplier", "second"), + 1200, + 300, + None, + None, + None, + True, + ), + ([_endpoint(supported_parameters=[])], ("supplier",), 1200, 300, None, None, None, False), + ( + [_endpoint(supported_parameters=None, context_length=None, max_completion_tokens=None)], + ("supplier",), + None, + None, + None, + None, + None, + None, + ), + ([_endpoint()], ("supplier/fp4",), None, None, None, None, None, None), + ([], ("supplier",), None, None, None, None, None, None), + ( + [_endpoint(), _endpoint("supplier/fast", context_length=10)], + ("supplier",), + 1200, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(), _endpoint("supplier/fast", context_length=10)], + ("supplier/fast",), + 10, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(status=-1, uptime_last_30m=0)], + ("supplier",), + 1200, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [ + _endpoint(reasoning_effort_levels=["high", "low"]), + _endpoint("second", reasoning_effort_levels=["max", "high"], default_reasoning_effort="max"), + ], + ("supplier", "second"), + 1200, + 300, + None, + ["high"], + None, + True, + ), + ), +) +async def test_pinned_metadata_guarantees_only_eligible_endpoint_contracts( + endpoints: list[Mapping[str, object]], + only: tuple[str, ...], + context: int | None, + output: int | None, + input_limit: int | None, + efforts: list[str] | None, + default: str | None, + reasoning: bool | None, +) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] == "fixture-key" + if request.url.path.endswith("/endpoints"): + assert request.url.path == "/custom/v1/models/org/model/endpoints" + return httpx.Response(200, json={"data": {"id": "org/model", "endpoints": endpoints}}) + assert request.url.path == "/custom/v1/models" + return httpx.Response(200, json=_catalog()) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + inventory, metadata = await _inventory(handler, InMemoryCache(), only=only) + assert metadata.get("context_window") == context + assert metadata.get("max_output_tokens") == output + assert metadata.get("max_input_tokens") == input_limit + assert metadata.get("reasoning_effort_levels") == efforts + assert metadata.get("default_reasoning_effort") == default + assert metadata.get("supports_reasoning") == reasoning + assert all(value is not None for value in metadata.values()) + assert get_inventory_model_metadata(inventory, "org/model")["context_window"] == 8000 + assert get_inventory_model_metadata(inventory, "org/model")["max_output_tokens"] == 7000 + + +@pytest.mark.parametrize( + ("reasoning", "expected"), + ( + ({}, {}), + ( + {"supported_efforts": None, "mandatory": True, "default_effort": "none"}, + {"reasoning_effort_levels": [level for level in OPENROUTER_REASONING_EFFORTS if level != "none"]}, + ), + ( + {"supported_efforts": ["high", "none"], "default_effort": "none", "default_enabled": False}, + {"reasoning_effort_levels": ["high", "none"], "default_reasoning_effort": "none"}, + ), + ({"supported_efforts": ["future-effort", "high"]}, {"reasoning_effort_levels": ["future-effort", "high"]}), + ( + {"supported_efforts": None, "default_effort": "high"}, + {"reasoning_effort_levels": list(OPENROUTER_REASONING_EFFORTS), "default_reasoning_effort": "high"}, + ), + ({"supported_efforts": [], "default_effort": "high"}, {"reasoning_effort_levels": []}), + ( + {"supported_efforts": ["xhigh", "high"], "default_effort": "xhigh", "default_enabled": False}, + {"reasoning_effort_levels": ["xhigh", "high"]}, + ), + ({"supported_efforts": ["max", "high"], "default_effort": "low"}, {"reasoning_effort_levels": ["max", "high"]}), + ), +) +async def test_model_reasoning_contract_is_not_invented_from_boolean_support( + reasoning: Mapping[str, object], expected: Mapping[str, object] +) -> None: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={"data": {"id": "org/model", "endpoints": [_endpoint()]}} + if request.url.path.endswith("/endpoints") + else _catalog(reasoning), + ) + ) + ) as client: + handler.client = client + _, metadata = await _inventory(handler, InMemoryCache()) + assert { + k: v for k, v in metadata.items() if k in ("reasoning_effort_levels", "default_reasoning_effort") + } == expected + + +@pytest.mark.parametrize( + "failure", + ( + httpx.Response(401), + httpx.Response(503), + httpx.Response(302, headers={"location": "https://foreign.test"}), + httpx.Response(200, content=b"bad json"), + httpx.Response(200, json={"data": {"id": "wrong/model", "endpoints": [_endpoint()]}}), + httpx.Response(200, json={"data": {"id": "org/model", "endpoints": [_endpoint(context_length=True)]}}), + httpx.Response(200, json={"data": {"id": "org/model", "endpoints": [_endpoint(max_prompt_tokens=-1)]}}), + httpx.Response(200, content=b"x" * (2 * 1024 * 1024 + 1)), + httpx.ReadTimeout("fixture timeout"), + ), +) +async def test_endpoint_failure_retains_same_identity_metadata_but_never_imports_aggregate_limits( + failure: httpx.Response | Exception, +) -> None: + endpoint_response: Final = Mock( + return_value=httpx.Response(200, json={"data": {"id": "org/model", "endpoints": [_endpoint()]}}) + ) + + def respond(request: httpx.Request) -> httpx.Response: + return endpoint_response() if request.url.path.endswith("/endpoints") else httpx.Response(200, json=_catalog()) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond), follow_redirects=True) as client: + handler.client = client + cache: Final = InMemoryCache() + first, metadata = await _inventory(handler, cache) + endpoint_response.side_effect = failure if isinstance(failure, Exception) else None + endpoint_response.return_value = failure + _, retained = await _inventory(handler, cache, refresh=True, previous=first) + assert retained == metadata + _, unavailable = await _inventory(handler, cache) + assert unavailable == {} + assert endpoint_response.call_count == 2 + _, other_identity = await _inventory(handler, cache, key="other-key", previous=first) + assert other_identity == {} + + +async def test_endpoint_cache_separates_models_credentials_and_bases_and_refreshes() -> None: + clock: Final = Mock(return_value=0) + responder: Final = Mock( + side_effect=lambda request: httpx.Response( + 200, + json={ + "data": { + "id": "org/second" if "/org/second/" in request.url.path else "org/model", + "endpoints": [_endpoint()], + } + } + if request.url.path.endswith("/endpoints") + else _catalog(), + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + cache: Final = InMemoryCache(clock=clock) + await _inventory(handler, cache) + await _inventory(handler, cache, only=("supplier",)) + assert responder.call_count == 2 + await _inventory(handler, cache, model="org/second") + assert responder.call_count == 3 + await _inventory(handler, cache, key="other") + assert responder.call_count == 5 + await _inventory(handler, cache, base="https://other.test/v1") + assert responder.call_count == 7 + await _inventory(handler, cache, refresh=True) + assert responder.call_count == 9 + clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1 + await _inventory(handler, cache) + assert responder.call_count == 11 + + +@pytest.mark.parametrize("endpoints", ([], [_endpoint("other")])) +async def test_successful_endpoint_absence_clears_last_known_metadata(endpoints: list[Mapping[str, object]]) -> None: + response: Final = Mock(return_value=[_endpoint()]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={"data": {"id": "org/model", "endpoints": response()}} + if request.url.path.endswith("/endpoints") + else _catalog(), + ) + ) + ) as client: + handler.client = client + cache: Final = InMemoryCache() + first, metadata = await _inventory(handler, cache) + assert metadata["context_window"] == 1200 + response.return_value = endpoints + current, absent = await _inventory(handler, cache, refresh=True, previous=first) + assert absent == {} + assert "org/model" in current.models + + +@pytest.mark.parametrize("model", ("org/../escape", "../model", "org/..")) +async def test_inventory_identifiers_cannot_change_endpoint_url_structure(model: str) -> None: + responder: Final = Mock(return_value=httpx.Response(200, json={"data": [{"id": model}]})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + _, metadata = await _inventory(handler, InMemoryCache(), model=model) + assert metadata == {} + assert responder.call_count == 1 diff --git a/tests/unit/proxy/test_model_discovery.py b/tests/unit/proxy/test_model_discovery.py index af0e63400e2..43c875f1121 100644 --- a/tests/unit/proxy/test_model_discovery.py +++ b/tests/unit/proxy/test_model_discovery.py @@ -2,8 +2,8 @@ from typing import Final import pytest -from litellm.proxy.model_discovery import discover_model_metadata, guaranteed_token_limit -from litellm.types.router import ModelMetadata, resolve_model_metadata +from litellm.proxy.model_discovery import discover_model_metadata +from litellm.types.router import ModelMetadata, guaranteed_token_limit, resolve_model_metadata def test_configured_overrides_preserve_false_empty_and_independent_limits(): diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py index 312b2a073ff..a5dd099927e 100644 --- a/tests/unit/proxy/test_model_offerings.py +++ b/tests/unit/proxy/test_model_offerings.py @@ -1,4 +1,5 @@ import json +from collections.abc import Mapping from pathlib import Path from types import MappingProxyType from typing import Final @@ -809,10 +810,12 @@ async def test_shared_inventory_client_is_reacquired_after_cache_eviction( await second.close() -@pytest.mark.parametrize("surface", ("chat", "responses", "responses_chat", "messages")) +@pytest.mark.parametrize("surface", ("chat", "chat_unified", "responses", "responses_chat", "messages")) @pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize("effort", (None, "high", "max")) +@pytest.mark.parametrize("source", ("manual", "auto")) async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, surface: str, stream: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, surface: str, stream: bool, effort: str | None, source: str ) -> None: path: Final = tmp_path / "offerings.yaml" policy: Final = {"only": ["synthetic/fp8"], "allow_fallbacks": False} @@ -830,7 +833,7 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( "offerings": [ { "model_name": "selected", - "source": "manual", + "source": source, "provider": "supplier", "upstream_model": "org/backend", "model_info": {"mode": "chat"}, @@ -844,12 +847,43 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( def respond(request: httpx.Request) -> httpx.Response: if request.url.path.endswith("/models"): - return httpx.Response(200, json={"data": []}) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/backend", + "reasoning": { + "supported_efforts": ["max", "high", "low"], + "default_effort": "high", + }, + } + ] + }, + ) + if request.url.path.endswith("/endpoints"): + return httpx.Response( + 200, + json={ + "data": { + "id": "org/backend", + "endpoints": [ + { + "tag": "synthetic/fp8", + "context_length": 1200, + "max_completion_tokens": 300, + "supported_parameters": ["reasoning", "reasoning_effort"], + } + ], + } + }, + ) requests.append(request) body: Final = json.loads(request.content) assert body["model"] == "org/backend" assert body["provider"] == policy assert body["temperature"] == 0.5 + assert (body.get("reasoning") or {}).get("effort", body.get("reasoning_effort")) == effort if request.url.path.endswith("/responses"): response: Final = { "id": "resp_fixture", @@ -934,10 +968,19 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( path=path, template=Router(model_list=[], num_retries=0), client=handler, clock=lambda: 0.0 ) assert await manager.reload(initial=True) + if source == "auto": + advertised: Final = manager.router.get_deployment_by_model_group_name("selected") + assert advertised is not None + assert advertised.model_info.reasoning_effort_levels == ("max", "high", "low") + assert advertised.model_info.default_reasoning_effort == "high" guard: Final = OfferingAccessGuard(manager.router) monkeypatch.setattr(litellm, "callbacks", [guard]) call_type: Final[CallTypesLiteral] = ( - "acompletion" if surface == "chat" else "anthropic_messages" if surface == "messages" else "aresponses" + "acompletion" + if surface in ("chat", "chat_unified") + else "anthropic_messages" + if surface == "messages" + else "aresponses" ) response_cache: Final = Cache( type="local", supported_call_types=["acompletion", "aresponses", "anthropic_messages"] @@ -949,11 +992,27 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( data={ "model": "selected", "stream": stream, - "extra_body": {"temperature": 0.5}, + "extra_body": { + "temperature": 0.5, + **({"reasoning": {"effort": effort}} if surface == "chat_unified" and effort is not None else {}), + }, "caching": True, "preset_cache_key": "shared-fixture-key", "cache": {"no-cache": False, "no-store": False}, **({"use_chat_completions_api": True} if surface == "responses_chat" else {}), + **( + ( + {"reasoning": {"effort": effort}} + if surface in ("responses", "responses_chat") + else {"thinking": {"type": "adaptive"}, "output_config": {"effort": effort}} + if surface == "messages" + else {} + if surface == "chat_unified" + else {"reasoning_effort": effort} + ) + if effort is not None + else {} + ), }, call_type=call_type, ) @@ -976,6 +1035,8 @@ async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( assert requests[0].url.path.endswith("/responses" if surface == "responses" else "/chat/completions") assert response_cache.cache.get_cache("shared-fixture-key") == {"response": "unrelated endpoint output"} for control in ( + {"service_tier": "priority"}, + {"extra_body": {"service_tier": "priority"}}, {"additional_drop_params": ["provider"]}, {"additional_drop_params": ["extra_body"]}, {"additional_drop_params": ["extra_body.provider"]}, @@ -1188,3 +1249,174 @@ async def test_endpoint_failure_never_uses_gateway_fallback_and_tampering_never_ disable_fallbacks=True, ) assert len(forwarded) == 1 + + +@pytest.mark.parametrize( + "overrides", + ( + {}, + {"context_window": 900, "max_output_tokens": 150, "max_input_tokens": 500}, + {"default_reasoning_effort": "max"}, + {"supports_reasoning": False, "supports_function_calling": False}, + {"reasoning_effort_levels": []}, + ), +) +async def test_auto_pinned_catalog_preserves_overrides_scope_and_endpoint_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, overrides: Mapping[str, object] +) -> None: + from litellm.proxy import proxy_server + + path: Final = tmp_path / "offerings.yaml" + selected: Final = { + "model_name": "openrouter/org/selected", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + "model_info": {"mode": "chat", **overrides}, + "provider_policy": {"provider": "openrouter", "only": ["supplier/fp8"], "allow_fallbacks": False}, + } + config: Final = { + "version": 1, + "providers": { + "supplier": {"provider": "openrouter", "api_base": "https://supplier.test/v1", "api_key": "fixture-key"} + }, + "offerings": [ + selected, + {**selected, "model_name": "ordinary", "provider_policy": None}, + {**selected, "model_name": "manual", "source": "manual"}, + ], + } + path.write_text(yaml.safe_dump(config)) + generations: Final = iter((1200, 700, None, None)) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] in ("Bearer fixture-key", "Bearer changed-key") + if request.url.path.endswith("/models"): + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/backend", + "context_length": 9000, + "top_provider": {"max_completion_tokens": 6000}, + "supported_parameters": ["tools", "reasoning_effort"], + "reasoning": {"supported_efforts": ["max", "high", "low"], "default_effort": "high"}, + }, + {"id": "org/unselected"}, + ] + }, + ) + assert request.url.path == "/v1/models/org/backend/endpoints" + context: Final = next(generations) + return ( + httpx.Response(503) + if context is None + else httpx.Response( + 200, + json={ + "data": { + "id": "org/backend", + "endpoints": [ + { + "tag": "supplier/fp8", + "context_length": context, + "max_completion_tokens": 300, + "max_prompt_tokens": None, + "supported_parameters": ["tools", "reasoning", "reasoning_effort"], + }, + {"tag": "foreign", "context_length": 9000, "max_completion_tokens": 6000}, + ], + } + }, + ) + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + monkeypatch.setattr(proxy_server, "llm_router", manager.router) + monkeypatch.setattr(proxy_server, "llm_model_list", manager.router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + application: Final = FastAPI() + application.include_router(proxy_server.router) + application.add_middleware(OfferingSnapshotMiddleware, router_getter=lambda: manager.router) + + async def scoped_auth(request: Request) -> UserAPIKeyAuth: + if request.headers.get("x-fixture-refresh"): + assert await manager.reload(force_inventory=True) + return UserAPIKeyAuth( + api_key="fixture-client", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["openrouter/*"] if request.headers.get("authorization") == "allowed" else ["denied/*"], + ) + + application.dependency_overrides[proxy_server.user_api_key_auth] = scoped_auth + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=application), base_url="http://gateway.test" + ) as gateway: + denied: Final = await gateway.get("/v1/models", headers={"authorization": "denied"}) + assert denied.status_code == 200 and denied.json()["data"] == [] + first: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert first.status_code == 200 + row: Final = first.json()["data"][0] + assert row["id"] == selected["model_name"] + assert row["context_window"] == overrides.get("context_window", 1200) + assert row["max_output_tokens"] == overrides.get("max_output_tokens", 300) + assert row.get("max_input_tokens") == overrides.get("max_input_tokens") + assert row["supports_function_calling"] == overrides.get("supports_function_calling", True) + if overrides.get("supports_reasoning") is False: + assert row["supports_reasoning"] is False + assert "reasoning_effort_levels" not in row and "default_reasoning_effort" not in row + else: + assert row["reasoning_effort_levels"] == overrides.get( + "reasoning_effort_levels", ["max", "high", "low"] + ) + assert row.get("default_reasoning_effort") == ( + None + if overrides.get("reasoning_effort_levels") == [] + else overrides.get("default_reasoning_effort", "high") + ) + assert all(value is not None for value in row.values()) + assert "supplier/fp8" not in first.text and "fixture-key" not in first.text + ordinary: Final = manager.router.get_deployment_by_model_group_name("ordinary") + manual: Final = manager.router.get_deployment_by_model_group_name("manual") + assert ordinary is not None and ordinary.model_info.context_window == overrides.get("context_window", 9000) + assert manual is not None and manual.model_info.context_window == overrides.get("context_window") + first_ids: Final = manager.router.get_model_ids(selected["model_name"]) + pinned: Final = await gateway.get( + "/v1/models", headers={"authorization": "allowed", "x-fixture-refresh": "1"} + ) + assert pinned.json() == first.json() + changed: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert changed.json()["data"][0]["context_window"] == overrides.get("context_window", 700) + assert manager.router.get_model_ids(selected["model_name"]) == first_ids + assert await manager.reload(force_inventory=True) + retained: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert retained.json() == changed.json() + path.write_text( + yaml.safe_dump( + { + **config, + "providers": { + "supplier": { + **config["providers"]["supplier"], + "api_key": "changed-key", + } + }, + } + ) + ) + assert await manager.reload() + rotated: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert rotated.json()["data"][0].get("context_window") == overrides.get("context_window") + assert rotated.json()["data"][0].get("reasoning_effort_levels") == overrides.get("reasoning_effort_levels") + assert manager.router.get_model_ids(selected["model_name"]) != first_ids diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py index d0b3b2623fd..b8f1d4ce3e4 100644 --- a/tests/unit/proxy/test_offering_router.py +++ b/tests/unit/proxy/test_offering_router.py @@ -278,6 +278,8 @@ async def test_body_model_override_cannot_bypass_offering_authority() -> None: {"provider_policy": {"provider": "openrouter", "only": ["other"]}}, {"extra_body": {"provider_policy": {"provider": "openrouter", "only": ["other"]}}}, {"provider": {"only": ["other"]}}, + {"service_tier": "priority"}, + {"extra_body": {"service_tier": "flex"}}, {"extra_body": {"provider": {"allow_fallbacks": True}}}, {"extra_body": {"extra_body": {"provider": {"only": ["other"]}}}}, {"models": ["other"]}, diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 6eb34d31f05..6e90696084b 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -60,6 +60,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/llms/litellm_proxy/skills/handler.py", "litellm/llms/openai/common_utils.py", "litellm/llms/openai_like/model_info.py", + "litellm/llms/openrouter/model_info.py", "litellm/llms/vertex_ai/vertex_ai_non_gemini.py", "litellm/llms/watsonx/common_utils.py", "litellm/proxy/_experimental/mcp_server/byok_credential_cache.py", From ae0b5efea9acd1c684ffdaf50c225c6424b6bbcf Mon Sep 17 00:00:00 2001 From: Hayden Moulds Date: Thu, 15 Oct 2026 17:38:50 +1000 Subject: [PATCH 28/34] fix(proxy): resolve model aliases to their target deployment for listing metadata (#41041) * fix(proxy): resolve model aliases to their target deployment for listing metadata A model alias (a key's `aliases`, a team's `model_aliases`, or the router's `model_group_alias`) is rewritten at request time and owns no deployment row, so `get_model_listing_info` returns None for it and the alias name is not a cost-map key either. Both metadata candidate sources come up empty and the listing entry carries no `mode`, `max_input_tokens`, or `max_output_tokens`, while the deployment it routes to reports them on the same response. Resolve a listed alias to the deployment it routes to for the metadata lookup, keeping the alias as the response id. Maps are applied in order, each against the result of the last, mirroring how `litellm_pre_call_utils` applies the team map and then the key map, so a chained alias reports the deployment the caller actually reaches rather than an intermediate one. An alias shadowing a same-named deployment follows the alias, because the request rewrite does too. Fixes the model_aliases half of #40553. * fix(proxy): look listed aliases up through the shared alias chain Drop the separate alias resolver and retarget a listed alias in alias_listing_entries and alias_target instead, so /v1/models and /v1/models/{id} report the model /chat/completions rewrites it to, and only when the caller can list that model * fix(proxy): keep deployed listed models' own rows and resolve team alias targets at list and retrieval * fix(proxy): resolve router_settings aliases, keep alias labels, and skip paused targets in model listing * fix(proxy): leave an alias to an undiscoverable model bare in model listing A team alias whose target has discoverable: false showed the hidden model's mode and limits on GET /v1/models and retrieved it by alias. Undiscoverable groups now count as hidden for alias resolution, as on main, while admins still see them. * fix(proxy): leave an alias to a model a listing callback hides bare in model listing * fix(proxy): keep aliases bare for models a listing callback never judged or another team owns --------- Co-authored-by: haydster7 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/common_utils/codex_model_catalog.py | 18 +- .../common_utils/discoverable_model_filter.py | 14 ++ .../proxy/common_utils/model_listing_utils.py | 196 ++++++++++++++--- litellm/proxy/proxy_server.py | 101 ++++++--- .../common_utils/test_model_listing_utils.py | 108 ++++++--- .../test_team_model_name_translation.py | 1 + tests/unit/proxy/test_model_list_aliases.py | 207 +++++++++++++++++- 7 files changed, 547 insertions(+), 98 deletions(-) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 115b31930dc..825b07b3c3c 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -376,14 +376,14 @@ def codex_models_response_json( def _catalog_row( - row: ModelInfoResponse, lookup_id: str, llm_router: Router | None, team_id: str | None + row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: return CodexCatalogRow( id=row["id"], mode=row.get("mode"), max_input_tokens=row.get("max_input_tokens"), upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None, - display_name=llm_router.get_configured_display_name(lookup_id) if llm_router is not None else None, + display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None, service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (), ) @@ -393,14 +393,21 @@ def codex_catalog_rows( entries: Sequence[tuple[str, str]], llm_router: Router | None, team_id: str | None = None, + label_entries: Sequence[tuple[str, str]] | None = None, ) -> tuple[CodexCatalogRow, ...]: """`rows` joined with the router's configured metadata, looked up by each entry's internal id so team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias` reading its target's deployments under its own name; `team_id` is the requesting key's team, so the upstream model that picks Codex's stock entry and the tiers are read only off the deployments its - requests can route to.""" + requests can route to. Display names are read under `label_entries`' ids when given.""" lookup_ids: Final = MappingProxyType(dict(entries)) - return tuple(_catalog_row(row, lookup_ids.get(row["id"], row["id"]), llm_router, team_id) for row in rows) + label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries)) + return tuple( + _catalog_row( + row, lookup_ids.get(row["id"], row["id"]), label_ids.get(row["id"], row["id"]), llm_router, team_id + ) + for row in rows + ) def codex_model_list_body( @@ -408,9 +415,10 @@ def codex_model_list_body( entries: Sequence[tuple[str, str]], llm_router: Router | None, team_id: str | None = None, + label_entries: Sequence[tuple[str, str]] | None = None, ) -> str: """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out.""" - body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id)) + body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries)) if body.left_out: verbose_proxy_logger.warning( "Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, " diff --git a/litellm/proxy/common_utils/discoverable_model_filter.py b/litellm/proxy/common_utils/discoverable_model_filter.py index d22f1a9f6dc..ca7368b8c72 100644 --- a/litellm/proxy/common_utils/discoverable_model_filter.py +++ b/litellm/proxy/common_utils/discoverable_model_filter.py @@ -109,6 +109,20 @@ def undiscoverable_model_names( ) +def undiscoverable_model_groups( + llm_router: Router | None, + user_api_key_dict: UserAPIKeyAuth, + team_id: str | None, +) -> frozenset[str]: + if llm_router is None or user_api_key_has_admin_view(user_api_key_dict): + return frozenset() + return frozenset( + row["model_name"] + for row in llm_router.get_model_list() or () + if is_undiscoverable_deployment(row) and is_undiscoverable_model_name(row["model_name"], llm_router, team_id) + ) + + def discoverable_rows( rows: Iterable[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 4c3d85fd4eb..4da20211b75 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Final, cast from pydantic import TypeAdapter, ValidationError import litellm +from litellm.router_utils.common_utils import resolve_model_group_alias if TYPE_CHECKING: from litellm.router import Router @@ -160,13 +161,47 @@ class ClaudeCodeRoutingNames: @dataclass(frozen=True, slots=True) class CallerAliases: - """`own` are the caller's key and team alias maps, the names `/v1/models` lists for it. - `rewrite` are the maps `/chat/completions` rewrites its model through, in the order it - applies them: the team's, the key's in `add_litellm_data_to_request`, then the global - `model_alias_map` and the key's again in `common_processing_pre_call_logic`.""" + """The caller's `team` and `key` alias maps, the names `/v1/models` lists for it, and `router`, + its key's or team's `router_settings.model_group_alias`. + `rewrite` are the maps `/chat/completions` rewrites its model through after auth's `router` + rewrite, in the order it applies them: the team's, the key's in `add_litellm_data_to_request`, + then the global `model_alias_map` and the key's again in `common_processing_pre_call_logic`, + which ends with `router` again.""" - own: tuple[object, ...] - rewrite: tuple[object, ...] + team: object + key: object + router: object = None + + @property + def own(self) -> tuple[object, ...]: + return (self.team, self.key) + + @property + def rewrite(self) -> tuple[object, ...]: + return (self.team, self.key, litellm.model_alias_map, self.key) + + +@dataclass(frozen=True, slots=True) +class DeployedModelGroups: + """Names the router serves `team_id` from deployments of their own, directly or as a + `model_group_alias`, leaving out the `hidden` names, groups `/v1/models` does not list, and, + when a listing callback runs, every name outside `callback_kept`, since it never judged those.""" + + llm_router: Router | None + hidden: Container[str] = frozenset[str]() + team_id: str | None = None + callback_kept: Container[str] | None = None + + def __contains__(self, name: object) -> bool: + if not isinstance(name, str) or self.llm_router is None or name in self.hidden: + return False + if self.callback_kept is not None and name not in self.callback_kept: + return False + group: Final = self.llm_router.routable_model_group(name) + return group not in self.hidden and any( + self.llm_router.deployment_usable_by_team(deployment, self.team_id) + for deployment in self.llm_router.get_all_deployments(model_name=group) + ) def caller_alias_maps( @@ -174,11 +209,12 @@ def caller_alias_maps( team_aliases: object, key_team_id: str | None, listed_team_id: str | None, + router_aliases: object = None, ) -> CallerAliases: """Team aliases count only when listing the team the key authenticated as.""" if listed_team_id is not None and listed_team_id != key_team_id: - return CallerAliases((key_aliases,), (key_aliases, litellm.model_alias_map, key_aliases)) - return CallerAliases((team_aliases, key_aliases), (team_aliases, key_aliases, litellm.model_alias_map, key_aliases)) + return CallerAliases(None, key_aliases, router_aliases) + return CallerAliases(team_aliases, key_aliases, router_aliases) def alias_map(aliases: object) -> Mapping[str, str]: @@ -191,42 +227,148 @@ def alias_map(aliases: object) -> Mapping[str, str]: ) +def _router_alias_map(aliases: object) -> Mapping[str, str]: + try: + entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True) + except ValidationError: + return _NO_ALIASES + targets: Final = ((alias, resolve_model_group_alias(entries, alias)) for alias in entries if isinstance(alias, str)) + return MappingProxyType({alias: target for alias, target in targets if target is not None}) + + +def _hidden_aliases(aliases: object) -> frozenset[str]: + try: + entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True) + except ValidationError: + return frozenset() + return frozenset( + alias + for alias, entry in entries.items() + if isinstance(alias, str) + and isinstance(entry, Mapping) + and cast(Mapping[str, object], entry).get("hidden") is True # cast-ok: isinstance-checked alias entry + ) + + def _alias_names(alias_maps: Sequence[Mapping[str, str]]) -> tuple[str, ...]: return tuple(dict.fromkeys(chain.from_iterable(alias_maps))) -def _rewrite(model_id: str, alias_maps: Sequence[Mapping[str, str]]) -> str | None: - target: Final = reduce(lambda name, aliases: aliases.get(name, name), alias_maps, model_id) - return None if target == model_id else target +@dataclass(frozen=True, slots=True) +class _AliasLayer: + aliases: Mapping[str, str] + checks_target: bool -def alias_target(model_id: str, aliases: CallerAliases, listed: Container[str] = frozenset()) -> str | None: - """The model group `/chat/completions` rewrites `model_id` to, else None. A `model_id` - already `listed` keeps its own row, so it is never rewritten.""" - if model_id in listed: - return None - return _rewrite(model_id, tuple(alias_map(raw) for raw in aliases.rewrite)) +def _alias_layers(aliases: CallerAliases) -> tuple[_AliasLayer, ...]: + """The maps `/chat/completions` rewrites its model through, in order, each marked when auth + checks the caller's access to the model it rewrites to instead of the requested one.""" + router: Final = _AliasLayer(_router_alias_map(aliases.router), checks_target=True) + key: Final = _AliasLayer(alias_map(aliases.key), checks_target=True) + return ( + router, + _AliasLayer(alias_map(aliases.team), checks_target=False), + key, + _AliasLayer(alias_map(litellm.model_alias_map), checks_target=False), + key, + router, + ) + + +def _through_layer(state: tuple[str, bool], layer: _AliasLayer) -> tuple[str, bool]: + name, checks_target = state + target: Final = layer.aliases.get(name, name) + return target, checks_target or (layer.checks_target and target != name) + + +def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]: + """The model `/chat/completions` sends `model_id` to, and whether auth checks the caller's + access to that model instead of `model_id`.""" + return reduce(_through_layer, layers, (model_id, False)) + + +def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None: + if name in lookup_by_response: + return lookup_by_response[name] + return name if name in lookup_ids else None + + +def _listed_lookup_id( + response_id: str, + lookup_id: str, + layers: Sequence[_AliasLayer], + lookup_by_response: Mapping[str, str], + lookup_ids: Container[str], + deployed: Container[str], +) -> str: + target, checks_target = _dispatch(response_id, layers) + if target == response_id or lookup_id in deployed: + return lookup_id + listed: Final = _lookup_id(target, lookup_by_response, lookup_ids) + if listed is not None: + return listed + return target if not checks_target and target in deployed else lookup_id + + +def _added_lookup_id( + alias: str, layers: Sequence[_AliasLayer], lookup_by_response: Mapping[str, str], lookup_ids: Container[str] +) -> str | None: + target, _ = _dispatch(alias, layers) + return None if target == alias else _lookup_id(target, lookup_by_response, lookup_ids) def alias_listing_entries( entries: Sequence[tuple[str, str]], aliases: CallerAliases, + deployed: Container[str], ) -> tuple[tuple[str, str], ...]: - """`entries` plus one `(alias, lookup_id)` row per key or team alias whose target is - listed. An alias colliding with a listed id keeps the listed entry.""" - maps: Final = tuple(alias_map(raw) for raw in aliases.rewrite) - own: Final = tuple(alias_map(raw) for raw in aliases.own) + """`entries` plus one row per key or team alias whose target is listed. A listed name + with no `deployed` group of its own is looked up under the model `/chat/completions` + sends it to when the caller may call that model through it, so the row describes the + model that answers.""" + layers: Final = _alias_layers(aliases) lookup_by_response: Final = MappingProxyType(dict(entries)) lookup_ids: Final = frozenset(lookup_by_response.values()) - targets: Final = MappingProxyType( - {alias: _rewrite(alias, maps) for alias in _alias_names(own) if alias not in lookup_by_response} + retargeted: Final = tuple( + (response_id, _listed_lookup_id(response_id, lookup_id, layers, lookup_by_response, lookup_ids, deployed)) + for response_id, lookup_id in entries ) added: Final = tuple( - (alias, lookup_by_response.get(target, target)) - for alias, target in targets.items() - if target is not None and (target in lookup_by_response or target in lookup_ids) + (alias, lookup_id) + for alias in _alias_names(tuple(alias_map(raw) for raw in aliases.own)) + if alias not in lookup_by_response + and (lookup_id := _added_lookup_id(alias, layers, lookup_by_response, lookup_ids)) is not None ) - return (*entries, *added) + return (*retargeted, *added) + + +def label_entries(listed: Sequence[tuple[str, str]], entries: Sequence[tuple[str, str]]) -> tuple[tuple[str, str], ...]: + """`entries` with every `listed` name read under its own lookup id again, so an alias + `alias_listing_entries` describes by its target keeps its own display name, the way a + `model_group_alias` does.""" + own: Final = MappingProxyType(dict(listed)) + return tuple((response_id, own.get(response_id, lookup_id)) for response_id, lookup_id in entries) + + +def alias_target( + model_id: str, + aliases: CallerAliases, + entries: Sequence[tuple[str, str]], + deployed: Container[str], +) -> str | None: + """The id `GET /v1/models/{model_id}` looks `model_id` up under when one of the caller's + aliases rewrites it, else None. A listed `model_id` resolves to the row + `alias_listing_entries` lists for it.""" + layers: Final = _alias_layers(aliases) + lookup_by_response: Final = MappingProxyType(dict(entries)) + listed_lookup_id: Final = lookup_by_response.get(model_id) + if listed_lookup_id is None: + target, _ = _dispatch(model_id, layers) + return None if target == model_id or model_id in _hidden_aliases(aliases.router) else target + lookup_id: Final = _listed_lookup_id( + model_id, listed_lookup_id, layers, lookup_by_response, frozenset(lookup_by_response.values()), deployed + ) + return None if lookup_id == listed_lookup_id else lookup_id def claude_code_requested_group( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 431292ec69c..81f842bd024 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -432,7 +432,11 @@ from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router -from litellm.proxy.common_utils.discoverable_model_filter import discoverable_rows, undiscoverable_model_names +from litellm.proxy.common_utils.discoverable_model_filter import ( + discoverable_rows, + undiscoverable_model_groups, + undiscoverable_model_names, +) from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -465,13 +469,14 @@ from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( ClaudeCodeRoutingNames, + DeployedModelGroups, TeamModelNameTranslator, alias_listing_entries, alias_target, caller_alias_maps, claude_code_view_ids, configured_display_names, - is_claude_code_client, + label_entries, ) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, @@ -7985,7 +7990,7 @@ class ProxyConfig: user_api_key_dict: UserAPIKeyAuth | None, prisma_client: PrismaClient | None, proxy_logging_obj: ProxyLogging | None = None, - ) -> dict | None: + ) -> Mapping[str, object] | None: return await self._get_hierarchical_router_settings(user_api_key_dict, prisma_client, proxy_logging_obj) async def _get_hierarchical_router_settings( @@ -11913,6 +11918,25 @@ async def _names_hidden_by_listing_callbacks( ) +def _describable_model_groups( + listed_entries: Sequence[tuple[str, str]], + hidden_names: frozenset[str] | set[str], + user_api_key_dict: UserAPIKeyAuth, + team_id: str | None, +) -> DeployedModelGroups: + listing_team_id: Final = team_id or user_api_key_dict.team_id + return DeployedModelGroups( + llm_router, + hidden_names | undiscoverable_model_groups(llm_router, user_api_key_dict, listing_team_id), + listing_team_id, + ( + frozenset(lookup_id for _, lookup_id in listed_entries) + if ProxyLogging.callback_capabilities().listed_models_filters + else None + ), + ) + + async def _entries_kept_by_listing_callbacks( entries: Sequence[tuple[str, str]], user_api_key_dict: UserAPIKeyAuth ) -> tuple[tuple[str, str], ...]: @@ -12000,21 +12024,20 @@ async def model_list( http_request is not None and http_request.headers.get("anthropic-version") is not None ) client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS - view_router_settings: Final = ( - await proxy_config.get_hierarchical_router_settings(user_api_key_dict, prisma_client, proxy_logging_obj) - if wants_anthropic_format and is_claude_code_client(client_headers) - else None - ) - view_aliases: Final = ( - view_router_settings.get("model_group_alias") if isinstance(view_router_settings, Mapping) else None + caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj ) caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, user_api_key_dict.team_model_aliases, user_api_key_dict.team_id, team_id + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + team_id, + caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, ) routing_names: Final = ClaudeCodeRoutingNames( llm_router, team_id or user_api_key_dict.team_id, - (*caller_aliases.rewrite, view_aliases), + (*caller_aliases.rewrite, caller_aliases.router), ) # Validate scope parameter if provided @@ -12037,7 +12060,7 @@ async def model_list( ) # Compute once — used in both branches below to hide paused models from the listing. - blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set() + blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set[str]() # Opt-in: also hide models whose deployments are all unhealthy per background # health checks. Empty when health state is unavailable or stale (fail open). @@ -12154,12 +12177,15 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - entries: Final = alias_listing_entries( - await _entries_kept_by_listing_callbacks( - TeamModelNameTranslator.listing_entries(all_models, llm_router, settings), user_api_key_dict - ), - caller_aliases, + listed_entries: Final = await _entries_kept_by_listing_callbacks( + TeamModelNameTranslator.listing_entries(all_models, llm_router, settings), user_api_key_dict ) + entries: Final = alias_listing_entries( + listed_entries, + caller_aliases, + _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + ) + labels: Final = label_entries(listed_entries, entries) for response_id, lookup_id in entries: model_info = create_model_info_response( model_id=lookup_id, @@ -12174,14 +12200,14 @@ async def model_list( listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id), + content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels), media_type="application/json", ) if wants_anthropic_format: return create_anthropic_model_list_response( listing, - display_names=configured_display_names(entries, llm_router), + display_names=configured_display_names(labels, llm_router), listed_ids=claude_code_view_ids( listing, client_headers, @@ -12251,7 +12277,7 @@ async def model_info( # Mirror /v1/models' visibility filter so first-occurrence resolution # cannot land on a deployment the listing had hidden. Undiscoverable # models stay retrievable by id, they only drop out of the alias guard. - blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set() + blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set[str]() unhealthy_names: Final = await get_hidden_unhealthy_model_names( healthy_only=healthy_only, general_settings=settings, @@ -12259,6 +12285,16 @@ async def model_info( ) hidden_names: Final = blocked_names | unhealthy_names internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, settings) + caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + caller_aliases: Final = caller_alias_maps( + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + team_id, + caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, + ) callback_hidden_names: Final = await _names_hidden_by_listing_callbacks( user_api_key_dict, tuple( @@ -12276,17 +12312,14 @@ async def model_info( all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id ) + listed_entries: Final = TeamModelNameTranslator.listing_entries( + tuple(m for m in all_models if m not in undiscoverable_names), llm_router, settings + ) aliased_model_id: Final = alias_target( model_id, - caller_alias_maps( - user_api_key_dict.aliases, user_api_key_dict.team_model_aliases, user_api_key_dict.team_id, team_id - ), - frozenset( - response_id - for response_id, _ in TeamModelNameTranslator.listing_entries( - tuple(m for m in all_models if m not in undiscoverable_names), llm_router, settings - ) - ), + caller_aliases, + listed_entries, + _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), ) resolved_model_id: Final = TeamModelNameTranslator.resolve_public_name( model_id=aliased_model_id or model_id, @@ -12295,14 +12328,16 @@ async def model_info( general_settings=settings, ) - # Validate that the requested model is accessible - validate_model_access(model_id=resolved_model_id, available_models=all_models) + if all(response_id != model_id for response_id, _ in listed_entries): + validate_model_access(model_id=resolved_model_id, available_models=all_models) # Get provider information from the router deployment if llm_router is None: raise HTTPException(status_code=500, detail="Router not initialized") - deployment: Final = llm_router.get_deployment_by_model_group_name(resolved_model_id) + deployment: Final = llm_router.get_deployment_by_model_group_name( + llm_router.routable_model_group(resolved_model_id) + ) if deployment is None: raise HTTPException( status_code=404, diff --git a/tests/unit/proxy/common_utils/test_model_listing_utils.py b/tests/unit/proxy/common_utils/test_model_listing_utils.py index 97e38d5e17a..3acaa7605a2 100644 --- a/tests/unit/proxy/common_utils/test_model_listing_utils.py +++ b/tests/unit/proxy/common_utils/test_model_listing_utils.py @@ -27,8 +27,8 @@ def _marked(name): return f"{_encoded(name)}[1m]" -def _caller(*maps: object) -> CallerAliases: - return CallerAliases(maps, maps) +def _caller(team: object = None, key: object = None) -> CallerAliases: + return CallerAliases(team, key) def _row(name, limit=1000000): @@ -134,42 +134,87 @@ def test_mutation_breaking_the_hex_name_cannot_route_to_the_source(source): def test_team_alias_is_listed_under_its_target_metadata_and_only_when_the_target_is_accessible() -> None: entries = [("gpt-4.1-mini", "gpt-4.1-mini"), ("team-public", "model_name_team_1_abc")] - aliases = ( + aliases = _caller( {"gpt-4.1-mini": "team-public"}, - None, {"claude-sonnet-4-5": "gpt-4.1-mini", "via-public": "team-public", "not-granted": "gpt-4.1"}, ) - assert alias_listing_entries(entries, _caller(*aliases)) == ( + deployed = frozenset({"gpt-4.1-mini", "model_name_team_1_abc", "gpt-4.1"}) + assert alias_listing_entries(entries, aliases, deployed) == ( *entries, ("claude-sonnet-4-5", "gpt-4.1-mini"), ("via-public", "model_name_team_1_abc"), ) - assert alias_listing_entries(entries, _caller(None, {})) == tuple(entries) + assert alias_listing_entries(entries, _caller(None, {}), deployed) == tuple(entries) def test_alias_target_resolves_the_requested_alias_across_key_and_team_maps() -> None: maps = _caller({"o": "gpt-4.1"}, {"claude-sonnet-4-5": "gpt-4.1-mini"}) - assert alias_target("claude-sonnet-4-5", maps) == "gpt-4.1-mini" - assert alias_target("gpt-4.1-mini", _caller(None, {"claude-sonnet-4-5": "gpt-4.1-mini"})) is None + assert alias_target("claude-sonnet-4-5", maps, (), frozenset()) == "gpt-4.1-mini" + assert alias_target("gpt-4.1-mini", _caller(None, {"claude-sonnet-4-5": "gpt-4.1-mini"}), (), frozenset()) is None -def test_alias_colliding_with_a_listed_id_keeps_the_listed_model_at_list_and_retrieval() -> None: +@pytest.mark.parametrize("layer", ["team", "key", "global"]) +def test_alias_colliding_with_a_deployed_listed_model_keeps_that_model_at_list_and_retrieval( + monkeypatch: pytest.MonkeyPatch, layer: str +) -> None: + alias = {"fast": "gpt-4.1-mini"} + monkeypatch.setattr(litellm, "model_alias_map", alias if layer == "global" else {}) + maps = _caller(alias if layer == "team" else None, alias if layer == "key" else None) entries = [("fast", "fast"), ("gpt-4.1-mini", "gpt-4.1-mini")] - maps = _caller({"fast": "gpt-4.1-mini"}) - listed = frozenset(response_id for response_id, _ in entries) + deployed = frozenset({"fast", "gpt-4.1-mini"}) - assert alias_listing_entries(entries, maps) == tuple(entries) - assert alias_target("fast", maps, listed) is None - assert alias_target("fast", maps) == "gpt-4.1-mini" + assert alias_listing_entries(entries, maps, deployed) == tuple(entries) + assert alias_target("fast", maps, entries, deployed) is None + assert alias_target("fast", maps, (), deployed) == "gpt-4.1-mini" + + +def test_listed_alias_is_looked_up_under_the_listed_model_it_rewrites_to_at_list_and_retrieval() -> None: + entries = [ + ("fast", "fast"), + ("gpt-4.1-mini", "gpt-4.1-mini"), + ("chat", "chat"), + ("internal", "internal"), + ("team-public", "model_name_team_1_abc"), + ] + maps = _caller({"fast": "gpt-4.1-mini", "chat": "team-public", "internal": "model_name_team_1_abc"}) + deployed = frozenset({"gpt-4.1-mini", "model_name_team_1_abc"}) + listing = alias_listing_entries(entries, maps, deployed) + + assert listing == ( + ("fast", "gpt-4.1-mini"), + ("gpt-4.1-mini", "gpt-4.1-mini"), + ("chat", "model_name_team_1_abc"), + ("internal", "model_name_team_1_abc"), + ("team-public", "model_name_team_1_abc"), + ) + assert all( + (alias_target(response_id, maps, entries, deployed) or dict(entries)[response_id]) == lookup_id + for response_id, lookup_id in listing + ) + + +def test_listed_team_alias_reports_its_deployed_target_unless_a_key_alias_needs_the_caller_to_list_it() -> None: + entries = [("fast", "fast"), ("gpt-4.1-mini", "gpt-4.1-mini")] + deployed = frozenset({"gpt-4.1-mini", "gpt-4.1"}) + through_team = _caller({"fast": "gpt-4.1"}) + through_key = _caller(None, {"fast": "gpt-4.1"}) + to_nothing_deployed = _caller({"fast": "restricted"}) + + assert alias_listing_entries(entries, through_team, deployed) == (("fast", "gpt-4.1"), entries[1]) + assert alias_target("fast", through_team, entries, deployed) == "gpt-4.1" + for maps in (through_key, to_nothing_deployed): + assert alias_listing_entries(entries, maps, deployed) == tuple(entries) + assert alias_target("fast", maps, entries, deployed) is None def test_alias_maps_apply_in_the_order_chat_completions_applies_them() -> None: team_then_key = _caller({"fast": "gpt-4.1-mini", "hop": "mid"}, {"fast": "gpt-4.1", "mid": "gpt-4.1"}) entries = [("gpt-4.1-mini", "gpt-4.1-mini"), ("gpt-4.1", "gpt-4.1")] + deployed = frozenset({"gpt-4.1-mini", "gpt-4.1"}) - assert alias_target("fast", team_then_key) == "gpt-4.1-mini" - assert alias_target("hop", team_then_key) == "gpt-4.1" - assert alias_listing_entries(entries, team_then_key) == ( + assert alias_target("fast", team_then_key, (), deployed) == "gpt-4.1-mini" + assert alias_target("hop", team_then_key, (), deployed) == "gpt-4.1" + assert alias_listing_entries(entries, team_then_key, deployed) == ( *entries, ("fast", "gpt-4.1-mini"), ("hop", "gpt-4.1"), @@ -180,17 +225,20 @@ def test_alias_maps_apply_in_the_order_chat_completions_applies_them() -> None: def test_one_bad_alias_entry_hides_only_itself() -> None: aliases = {"fast": "gpt-4.1-mini", "broken": 5, 7: "gpt-4.1-mini"} entries = [("gpt-4.1-mini", "gpt-4.1-mini")] + deployed = frozenset({"gpt-4.1-mini"}) - assert alias_listing_entries(entries, _caller(aliases)) == (*entries, ("fast", "gpt-4.1-mini")) - assert alias_target("fast", _caller(aliases)) == "gpt-4.1-mini" + assert alias_listing_entries(entries, _caller(aliases), deployed) == (*entries, ("fast", "gpt-4.1-mini")) + assert alias_target("fast", _caller(aliases), (), deployed) == "gpt-4.1-mini" def test_chained_key_alias_is_listed_only_when_its_final_target_is_listable() -> None: key_aliases = {"a": "b", "b": "hidden"} entries = [("b", "b")] + maps = caller_alias_maps(key_aliases, None, "team-a", None) + deployed = frozenset({"b", "hidden"}) - assert alias_listing_entries(entries, caller_alias_maps(key_aliases, None, "team-a", None)) == (*entries,) - assert alias_target("a", caller_alias_maps(key_aliases, None, "team-a", None)) == "hidden" + assert alias_listing_entries(entries, maps, deployed) == (*entries,) + assert alias_target("a", maps, (), deployed) == "hidden" def test_team_aliases_only_apply_when_listing_the_team_the_key_authenticated_as( @@ -198,12 +246,10 @@ def test_team_aliases_only_apply_when_listing_the_team_the_key_authenticated_as( ) -> None: key_aliases, team_aliases, global_aliases = {"k": "gpt-4.1"}, {"t": "gpt-4.1-mini"}, {"g": "gpt-4.1"} monkeypatch.setattr(litellm, "model_alias_map", global_aliases) - own_team = CallerAliases((team_aliases, key_aliases), (team_aliases, key_aliases, global_aliases, key_aliases)) - assert caller_alias_maps(key_aliases, team_aliases, "team-a", None) == own_team + own_team = caller_alias_maps(key_aliases, team_aliases, "team-a", None) + assert own_team.rewrite == (team_aliases, key_aliases, global_aliases, key_aliases) assert caller_alias_maps(key_aliases, team_aliases, "team-a", "team-a") == own_team - assert caller_alias_maps(key_aliases, team_aliases, "team-a", "team-b") == CallerAliases( - (key_aliases,), (key_aliases, global_aliases, key_aliases) - ) + assert caller_alias_maps(key_aliases, team_aliases, "team-a", "team-b") == CallerAliases(None, key_aliases) def test_global_alias_rewrites_between_the_two_key_passes_like_chat_completions( @@ -213,18 +259,20 @@ def test_global_alias_rewrites_between_the_two_key_passes_like_chat_completions( key_aliases = {"a": "b", "b": "c"} entries = [("c", "c"), ("d", "d")] maps = caller_alias_maps(key_aliases, None, "team-a", None) + deployed = frozenset({"c", "d"}) - assert alias_target("a", maps) == "d" - assert alias_listing_entries(entries, maps) == (*entries, ("a", "d"), ("b", "c")) + assert alias_target("a", maps, (), deployed) == "d" + assert alias_listing_entries(entries, maps, deployed) == (*entries, ("a", "d"), ("b", "c")) def test_global_aliases_rewrite_but_are_not_listed_as_caller_rows(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_alias_map", {"g": "gpt-4.1-mini"}) entries = [("gpt-4.1-mini", "gpt-4.1-mini")] maps = caller_alias_maps({"k": "g"}, None, "team-a", None) + deployed = frozenset({"gpt-4.1-mini"}) - assert alias_listing_entries(entries, maps) == (*entries, ("k", "gpt-4.1-mini")) - assert alias_target("g", maps) == "gpt-4.1-mini" + assert alias_listing_entries(entries, maps, deployed) == (*entries, ("k", "gpt-4.1-mini")) + assert alias_target("g", maps, (), deployed) == "gpt-4.1-mini" def test_team_public_name_uses_the_same_scope_at_list_and_request(): diff --git a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py index a850b3a0d83..1e2f0596048 100644 --- a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py @@ -1420,6 +1420,7 @@ def _public_named_router(*team_rows: dict) -> MagicMock: router = MagicMock() router.get_model_list.return_value = list(team_rows) router.get_model_listing_info.return_value = None + router.routable_model_group.side_effect = lambda model_name: model_name return router diff --git a/tests/unit/proxy/test_model_list_aliases.py b/tests/unit/proxy/test_model_list_aliases.py index 25a941bf2ae..0ec35a82e8d 100644 --- a/tests/unit/proxy/test_model_list_aliases.py +++ b/tests/unit/proxy/test_model_list_aliases.py @@ -4,12 +4,19 @@ Tests for key and team `model_aliases` on the model listing endpoints: GET /v1/m An alias the caller can complete on is listed next to its target and resolves by name. """ +import json +from collections.abc import Sequence + import pytest +from fastapi import HTTPException from starlette.requests import Request +import litellm from litellm import Router +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.utils import ProxyLogging def _deployment(model_name: str, model: str = "openai/gpt-4.1-mini", **model_info: str | bool) -> dict[str, object]: @@ -20,15 +27,41 @@ def _deployment(model_name: str, model: str = "openai/gpt-4.1-mini", **model_inf } +class _ListingGate(CustomLogger): + def __init__(self, hidden: frozenset[str]) -> None: + super().__init__() + self.hidden = hidden + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + return [name for name in model_names if name not in self.hidden] + + +class _ListingAllowList(CustomLogger): + def __init__(self, kept: frozenset[str]) -> None: + super().__init__() + self.kept = kept + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + return [name for name in model_names if name in self.kept] + + @pytest.fixture def router(monkeypatch: pytest.MonkeyPatch) -> Router: router = Router( model_list=[ _deployment("gpt-4.1-mini"), - _deployment("gpt-4.1", model="openai/gpt-4.1"), - _deployment("model_name_team1_abc", team_id="team1", team_public_model_name="team-chat"), + _deployment("gpt-4.1", model="openai/gpt-4.1", display_name="GPT 4.1"), + _deployment( + "model_name_team1_abc", model="openai/gpt-4o", team_id="team1", team_public_model_name="team-chat" + ), _deployment("hidden", model="anthropic/claude-sonnet-4-5", discoverable=False), - ] + _deployment("paused", model="openai/gpt-4.1", blocked=True), + ], + model_group_alias={"legacy": "gpt-4.1"}, ) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) @@ -149,3 +182,171 @@ async def test_v1_models_by_id_keeps_the_alias_as_id_when_it_targets_a_team_scop assert "chat" in await _v1_models(caller) response = await proxy_server.model_info(model_id="chat", user_api_key_dict=caller) assert response["id"] == "chat" + + +@pytest.mark.asyncio +async def test_alias_the_key_also_lists_reports_its_target_metadata_at_list_and_retrieval(router: Router) -> None: + caller = _team_member( + models=["gpt-4.1-mini", "model_name_team1_abc", "fast"], team_model_aliases={"fast": "gpt-4.1-mini"} + ) + + response = await proxy_server.model_list(user_api_key_dict=caller, request=None) + rows = {row["id"]: row for row in response["data"]} + assert rows["fast"] == {**rows["gpt-4.1-mini"], "id": "fast"} + target = await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + + +def _keyholder(*models: str) -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.INTERNAL_USER, models=list(models)) + + +@pytest.mark.asyncio +async def test_team_alias_to_a_model_the_team_does_not_list_reports_that_model_at_list_and_retrieval( + router: Router, +) -> None: + caller = _team_member(models=["gpt-4.1-mini", "model_name_team1_abc", "fast"], team_model_aliases={"fast": "gpt-4.1"}) + target_holder = _keyholder("gpt-4.1") + + rows = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + target_rows = (await proxy_server.model_list(user_api_key_dict=target_holder, request=None))["data"] + assert [rows["fast"]] == [{**row, "id": "fast"} for row in target_rows if row["id"] == "gpt-4.1"] + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=target_holder) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + + +@pytest.mark.asyncio +async def test_alias_named_like_a_listed_model_keeps_that_model_s_metadata_at_list_and_retrieval( + router: Router, +) -> None: + plain = await proxy_server.model_list(user_api_key_dict=_team_member(), request=None) + shadowing = _team_member(team_model_aliases={"gpt-4.1-mini": "team-chat"}) + + assert (await proxy_server.model_list(user_api_key_dict=shadowing, request=None))["data"] == plain["data"] + assert await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=shadowing) == ( + await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=_team_member()) + ) + + +@pytest.mark.asyncio +async def test_router_alias_the_caller_lists_resolves_at_retrieval_like_its_listing_row(router: Router) -> None: + caller = _keyholder("gpt-4.1", "legacy") + + rows = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + assert rows["legacy"] == {**rows["gpt-4.1"], "id": "legacy"} + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="legacy", user_api_key_dict=caller) == {**target, "id": "legacy"} + + +def _alias_holder(**aliases: dict[str, object]) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-test", + user_id="u", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["gpt-4.1", "fast"], + **aliases, + ) + + +@pytest.mark.asyncio +async def test_listed_alias_keeps_its_own_display_name_in_the_anthropic_and_codex_listings(router: Router) -> None: + caller = _alias_holder(aliases={"fast": "gpt-4.1"}) + + anthropic = await proxy_server.model_list(user_api_key_dict=caller, request=_anthropic_request()) + assert {row["id"]: row["display_name"] for row in anthropic["data"]} == {"gpt-4.1": "GPT 4.1", "fast": "fast"} + codex = await proxy_server.model_list(user_api_key_dict=caller, request=None, client_version="0.200.0") + target, alias = json.loads(codex.body)["models"] + assert (target["display_name"], alias["display_name"]) == ("GPT 4.1", "fast") + assert {**alias, "slug": "gpt-4.1", "display_name": "GPT 4.1", "priority": 0} == target + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_group_alias", [{"fast": "gpt-4.1"}, {"fast": {"model": "gpt-4.1", "hidden": True}}] +) +async def test_key_router_settings_alias_the_key_lists_reports_its_target_at_list_and_retrieval( + router: Router, model_group_alias: dict[str, object] +) -> None: + caller = _alias_holder(router_settings={"model_group_alias": model_group_alias}) + + rows = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + assert rows["fast"] == {**rows["gpt-4.1"], "id": "fast"} + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("target", "team_id"), [("paused", "team1"), ("hidden", "team1"), ("model_name_team1_abc", "team2")] +) +async def test_alias_to_a_model_the_listing_hides_is_not_described_by_it( + router: Router, target: str, team_id: str +) -> None: + models = ["gpt-4.1-mini", "fast"] + caller = _team_member(team_id=team_id, models=models, team_model_aliases={"fast": target}) + + listing = await proxy_server.model_list(user_api_key_dict=caller, request=None, include_metadata=True) + unaliased = await proxy_server.model_list( + user_api_key_dict=_team_member(team_id=team_id, models=models), request=None, include_metadata=True + ) + assert listing["data"] == unaliased["data"] + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) + assert retrieval.value.status_code == 404 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("target", "gated", "models"), + [ + ("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast"]), + ("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast", "gpt-4.1"]), + ("legacy", "legacy", ["gpt-4.1-mini", "fast"]), + ("legacy", "gpt-4.1", ["gpt-4.1-mini", "fast"]), + ("model_name_team1_abc", "team-chat", ["gpt-4.1-mini", "fast", "model_name_team1_abc"]), + ], +) +async def test_alias_to_a_model_a_listing_callback_hides_is_not_described_by_it( + router: Router, monkeypatch: pytest.MonkeyPatch, target: str, gated: str, models: list[str] +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_ListingGate(frozenset({gated}))]) + ProxyLogging._callback_capabilities_cache.clear() + caller = _team_member(models=models, team_model_aliases={"fast": target}) + + listing = await proxy_server.model_list(user_api_key_dict=caller, request=None, include_metadata=True) + unaliased = await proxy_server.model_list( + user_api_key_dict=_team_member(models=models), request=None, include_metadata=True + ) + assert listing["data"] == unaliased["data"] + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) + assert retrieval.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_allow_list_callback_keeps_every_name_it_allows_when_an_alias_points_past_them( + router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + caller = _team_member(models=["gpt-4.1-mini", "legacy", "fast"], team_model_aliases={"fast": "gpt-4.1"}) + listed = await _v1_models(caller) + + monkeypatch.setattr(litellm, "callbacks", [_ListingAllowList(frozenset(listed))]) + ProxyLogging._callback_capabilities_cache.clear() + assert await _v1_models(caller) == listed + + +@pytest.mark.asyncio +async def test_hidden_router_settings_alias_the_key_does_not_list_is_not_retrievable(router: Router) -> None: + caller = UserAPIKeyAuth( + api_key="sk-test", + user_id="u", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["gpt-4.1"], + router_settings={"model_group_alias": {"fast": "gpt-4.1", "quiet": {"model": "gpt-4.1", "hidden": True}}}, + ) + + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="quiet", user_api_key_dict=caller) + assert retrieval.value.status_code == 404 From ec1e681d95dca0e6660e21fba9060ec9e3c7afde Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 15 Oct 2026 13:25:55 +0200 Subject: [PATCH 29/34] fix(proxy): adapt offering admission to native alias listing context --- litellm/proxy/offering_router.py | 32 +++++++++++-------- ...al_pass_through_adapters_transformation.py | 6 ++-- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py index eb0e41c688b..680208ed74e 100644 --- a/litellm/proxy/offering_router.py +++ b/litellm/proxy/offering_router.py @@ -13,7 +13,13 @@ from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.llms.chatgpt.authenticator import prevent_device_login from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator, alias_target, caller_alias_maps +from litellm.proxy.common_utils.model_listing_utils import ( + DeployedModelGroups, + TeamModelNameTranslator, + alias_listing_entries, + alias_target, + caller_alias_maps, +) from litellm.router import Router from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.utils import CallTypes, CallTypesLiteral @@ -153,19 +159,19 @@ class OfferingAccessGuard(CustomLogger): def _offering_name(self, name: str, auth: UserAPIKeyAuth) -> str: snapshot: Final = self.router.serving_snapshot() auth_metadata: Final = _METADATA_ADAPTER.validate_python(auth.model_dump()) - caller_target: Final = ( - alias_target( - name, - caller_alias_maps( - _METADATA_ADAPTER.validate_python(auth_metadata.get("aliases") or MappingProxyType({})), - _METADATA_ADAPTER.validate_python(auth_metadata.get("team_model_aliases") or MappingProxyType({})), - auth.team_id, - auth.team_id, - ), - snapshot.available_models | frozenset(snapshot.unavailable_models), - ) - or name + aliases: Final = caller_alias_maps( + _METADATA_ADAPTER.validate_python(auth_metadata.get("aliases") or MappingProxyType({})), + _METADATA_ADAPTER.validate_python(auth_metadata.get("team_model_aliases") or MappingProxyType({})), + auth.team_id, + auth.team_id, ) + deployed: Final = DeployedModelGroups(snapshot.router, team_id=auth.team_id) + entries: Final = alias_listing_entries( + tuple((name, name) for name in snapshot.available_models | frozenset(snapshot.unavailable_models)), + aliases, + deployed, + ) + caller_target: Final = alias_target(name, aliases, entries, deployed) or dict(entries).get(name, name) target: Final = resolve_model_group_alias(snapshot.router.model_group_alias, caller_target) or caller_target rows: Final = TypeAdapter(tuple[Mapping[str, object], ...]).validate_python( snapshot.router.get_model_list() or () diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2eee8549c0b..921a5544271 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -4719,7 +4719,7 @@ def test_completion_cost_on_translated_anthropic_response_includes_web_search(): "model, provider, carried", [ ("databricks/databricks-claude-opus-4-7", "databricks", "max"), - ("openrouter/anthropic/claude-opus-4.7", "openrouter", "xhigh"), + ("openrouter/anthropic/claude-opus-4.7", "openrouter", "max"), ], ) def test_a_summary_bearing_adaptive_request_still_delivers_its_tier(model, provider, carried): @@ -4729,8 +4729,8 @@ def test_a_summary_bearing_adaptive_request_still_delivers_its_tier(model, provi exactly the path this translator exists to serve. Each case names the exact tier that provider ends up sending, not merely that something arrived: - bedrock and databricks rebuild `output_config`, and openrouter applies its own max to xhigh - remap, so asserting presence alone would pass on a mapping that silently changed the tier.""" + bedrock and databricks rebuild `output_config`, while OpenRouter keeps its gateway effort + unchanged, so asserting presence alone would pass on a mapping that silently changed the tier.""" from litellm.types.llms.anthropic import AnthropicMessagesRequest from litellm.utils import get_optional_params From 7011340e8c6d33355c76b8643f9d0ee507b7c908 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Thu, 15 Oct 2026 14:36:32 +0200 Subject: [PATCH 30/34] docs(offerings): explain metadata authority and the native serving flow --- examples/model_offerings/ARCHITECTURE.md | 221 +++++++++++++++++++++++ examples/model_offerings/README.md | 3 +- 2 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 examples/model_offerings/ARCHITECTURE.md diff --git a/examples/model_offerings/ARCHITECTURE.md b/examples/model_offerings/ARCHITECTURE.md new file mode 100644 index 00000000000..c40d0ad6022 --- /dev/null +++ b/examples/model_offerings/ARCHITECTURE.md @@ -0,0 +1,221 @@ +# How selected offerings become a client catalog + +Automatic supplier metadata is the normal path for AUTO offerings. Selecting an OpenRouter endpoint is a routing decision, not an instruction to replace discovery with handwritten capabilities. Keep overrides for an intentional operator policy or an evidenced missing value, not merely because a model was configured + +The schema still requires an explicit `source`. Normal automatic usage selects `source: auto` without copying a capability card; it does not mean an omitted `source` field is accepted + +This walkthrough is verified against runtime commit `ec1e681d95dca0e6660e21fba9060ec9e3c7afde`. Relative source links follow the code in the same checkout. The feature extends native LiteLLM adapters, types, authentication, listing and Router paths; referencing those paths does not mean every implementation in them originated in this PR. See the [setup guide](README.md) for configuration and lifecycle instructions + +```text +supplier connection -> inventory -> native normalization -> offering resolution + -> Router generation -> authorized gateway catalog + -> client options -> request mapping -> supplier +``` + +## 1. Connection, offering and metadata are different layers + +The [configuration types](../../litellm/types/proxy/model_offerings.py) separate these concerns + +| Layer | What it owns | What it does not imply | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Supplier connection | Native provider/protocol, trusted or configured API base, authentication and headers | That every model at the supplier becomes public | +| Selected offering | Public `model_name`, `upstream_model`, connection reference, enabled state, AUTO/MANUAL source and optional provider policy | That the operator must write a capability card or implement a provider plugin | +| Metadata and request defaults | Limits, capabilities, effort choices/default, API surfaces/modalities and optional output-budget policy | That routing selection, UI choices and physical limits are interchangeable | + +The `provider` on an offering references a connection name. That connection's `provider` selects an existing adapter, currently `chatgpt`, `openrouter`, `vercel_ai_gateway` or generic `openai` in this external-file schema. A custom connection name or public alias does not create custom provider code. A service that already speaks the supported OpenAI-compatible protocol can use the generic adapter; new provider implementation is needed only when its authentication, protocol or request/response behavior actually requires one + +## 2. Retrieve an inventory, without exposing it wholesale + +The [native inventory dispatcher](../../litellm/llms/model_inventory.py) chooses the supplier path. Discovery happens on the gateway using the selected connection's identity. A client does not need a second connection or credentials for the supplier + +| Connection | Inventory and normalization path | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Native ChatGPT account | The [ChatGPT adapter](../../litellm/llms/chatgpt/model_info.py) reads the saved server OAuth account and calls the trusted backend's `/models?client_version=...`. An explicit base must match that backend. Discovery requests existing access tokens with device login disabled | +| OpenRouter | The [OpenAI-compatible inventory](../../litellm/llms/openai_like/model_info.py) fetches the configured models catalog. Pinned AUTO offerings additionally use the [OpenRouter endpoint adapter](../../litellm/llms/openrouter/model_info.py) for `/models/{author}/{slug}/endpoints` under that API base | +| Vercel AI Gateway | Uses the compatible catalog request, then interprets Vercel's model kind, modalities, reasoning options and language-model output-budget field | +| Generic compatible service | Uses the configured `/v1/models` catalog and recognized metadata fields, without guessing a model family from its name | + +`add_path_to_api_base` handles an API base that already includes its version path. Discovery does not blindly append another `/v1` to it. Provider-specific native getters resolve OpenRouter/Vercel bases rather than maintaining a second URL registry + +Endpoint requests are deduplicated and restricted to enabled, selected AUTO models with a policy and a successful model-inventory membership result. The implementation does not request endpoint details for every model in a large supplier catalog. MANUAL metadata resolution ignores inventory, although the manager may still refresh configured connections + +Three decisions remain separate: a supplier reports a model, the operator selects and enables its public offering, and a caller is authorized to discover/use it. Downloading a model does not automatically enable it or grant it to any key + +## 3. Normalize meaning, not just field names + +[ModelMetadata and ModelRequestDefaults](../../litellm/types/router.py) provide the shared native facet used by inventory, deployments and listing + +| Field | Meaning | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `context_window` | Combined input and generated-output token capacity | +| `max_input_tokens` | Independently known input ceiling, not context minus output | +| `max_output_tokens` | Independently known generated-output ceiling, not a requested default | +| `supports_function_calling`, `supports_parallel_function_calling`, `supports_reasoning` | Known capabilities, including an explicit false when the source supports that conclusion | +| `reasoning_effort_levels` | Exact accepted effort choices, not a guessed low/medium/high ladder | +| `default_reasoning_effort` | A compatible advertised default or normalized API-facing default, subject to the provider qualifications below | +| `supported_endpoints`, `supported_modalities`, `supported_output_modalities` | Known API surfaces and input/output modalities | +| `request_defaults` | Optional requested output budgets, separate from all physical limits | + +Unknown optional fields are omitted after applicable sources are resolved. False and an empty list remain meaningful values. Numeric zero in supplier limits is treated as unknown; malformed supplier metadata is not evidence of a successful empty inventory. Validation also prevents an effort default outside a known choice list and clears choices/defaults when reasoning is explicitly unsupported + +### Provider-specific interpretations + +The generic card adapter recognizes native fields and context aliases such as `max_model_len` and `context_length`. For unpinned OpenRouter cards it also reads `top_provider`, architecture and model-level reasoning information. For Vercel, `type` can supply chat/embedding mode, `reasoning_options` supplies effort choices, and `max_tokens` is treated as output capacity only for a language model, not blindly for embeddings + +Native ChatGPT's Codex `context_window` and `max_context_window` are retained as `codex_context_window` and `codex_max_context_window` in inventory/deployment metadata. They are not unconditionally promoted to generic physical `context_window` or independent input capacity. Independently reported input/output fields can still be used. These Codex-specific extras are not part of the generic `/v1/models` projection, so their retention is not a promise that every client receives them + +ChatGPT discovery removes `ultra` from API effort choices because it is a multi-agent UI choice. It translates the catalog spelling `persistent` to advertised API spelling `disabled`. For a native default of `ultra`, the current normalization chooses a supported `multi_agent_reasoning_effort`, otherwise supported `max`, otherwise the last remaining reported API effort, otherwise omits the default. This is an explicit UI-to-API normalization policy, not proof that multi-agent Ultra is equivalent to a particular single-request effort or that the fallback is a measured backend default + +OpenRouter effort choices describe OpenRouter's gateway contract, not an independently verified native enum for a supplier such as StreamLake. Explicit `reasoning.supported_efforts: null` expands to the documented gateway effort set; omission does not declare choices; `[]` remains empty. Mandatory reasoning excludes `none`. A disabled-by-default model does not claim an active default effort, while an explicitly reported compatible `none` default can remain. Boolean reasoning support alone never creates choices + +## 4. Resolve AUTO, pinned AUTO and MANUAL + +The precedence is implemented by `_deployment` in the [offering manager](../../litellm/proxy/model_offerings.py), not by a client-side table + +| Mode | Availability | Metadata resolution, lowest to highest priority | Missing data | +| ----------------------- | ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| AUTO, unpinned | Enabled and present in the connection's latest successful model inventory | Provider-appropriate bundled metadata fallback, then downloaded normalized supplier fields, then explicit non-null `model_info` fields | Missing supplier fields can still have known bundled fallback. Remaining unknown fields are omitted | +| AUTO, pinned OpenRouter | Same model-membership rule, with endpoint policy retained | Metadata from eligible endpoints plus applicable model-level reasoning information, then explicit non-null `model_info` fields | No unrestricted aggregate or bundled capacity substitution. Unknown endpoint guarantees stay omitted | +| MANUAL | Enabled, independent of supplier inventory membership | Handwritten `model_info` only | Missing metadata remains unknown. The existing protocol adapter can still perform inference | + +For unpinned AUTO fallback, the manager looks up the provider-prefixed model in the native loaded cost map and permits an unprefixed fallback for generic `openai`. It reuses the native catalog rather than building another pricing/model registry. Consequently, a field on an AUTO GPT offering need not have come directly from the latest live JSON. Unpinned OpenRouter metadata is a model-level card; it does not certify which endpoint will serve an individual future request + +For pinned AUTO, `OpenRouterProviderPolicy.allows_endpoint` matches full slugs and base/variant prefixes, with service-tier suffixes requiring explicit selection. For every independently known limit, the common native aggregator takes the minimum across eligible endpoints. A limit unknown on any eligible endpoint stays unknown. Capability guarantees and lists are combined conservatively, and a default is retained only when compatible across sources. Status, uptime, price and performance do not change this metadata eligibility calculation or operator availability policy + +Effort choices/defaults are combined with endpoint metadata only where eligible endpoints declare `reasoning_effort` support. More restrictive endpoint declarations narrow model-level choices. An endpoint permitting reasoning without the effort control does not justify advertising an effort selector. Missing/failed endpoint discovery never borrows a larger context from another supplier + +### What an override actually does + +An explicit valid, non-null field overrides that field, not the entire downloaded card. A false capability or `[]` choice list survives the merge, subject to native consistency validation. `null` is omitted by the configured metadata dump and therefore does not clear an inherited value. A supplied `request_defaults` object replaces that field as an object; this is not a recursive merge of its budget map + +The override facility is optional. Copied endpoint values used as a temporary workaround are not automatically an intentional long-term policy. Remove unnecessary copied limit/capability fields from an AUTO definition to let discovery supply and refresh them; retain genuine operator policy and required configuration such as pinned `mode: chat`. Keeping the facility does not require keeping a workaround forever + +For a sole eligible endpoint reporting context `300000` and output `30000`, AUTO without intentional overrides advertises those bounds. If it later reports `280000` and `24000`, the next successful refresh publishes those values for subsequent requests. An unknown independent input ceiling remains omitted in both generations. These are illustrative numbers, not a claim about a named supplier + +## 5. Configure selection without copying a card + +These snippets illustrate standalone external-file definitions. Replace placeholder model IDs with exact IDs from the relevant catalog. They do not assert current capabilities for a vendor model or modify an existing deployment + +### Native GPT offering, automatically described + +```yaml +version: 1 +providers: + account: + provider: chatgpt +offerings: + - model_name: gpt-auto + provider: account + upstream_model: catalog-model-slug + source: auto + enabled: true +``` + +No capability block is needed to enable discovery. The account must already be authorized through the native server mechanism. Add a metadata field only for an intentional exception or evidenced information the automatic sources do not provide + +### Endpoint selection, still automatically described + +```yaml +version: 1 +providers: + aggregator: + provider: openrouter + api_key: os.environ/OPENROUTER_API_KEY +offerings: + - model_name: pinned-auto + provider: aggregator + upstream_model: organization/model + source: auto + enabled: true + provider_policy: + provider: openrouter + only: [supplier/fp8] + allow_fallbacks: false + model_info: + mode: chat +``` + +The current schema requires the matching OpenRouter supplier and explicit chat mode for a policy. It does not require static context, output or reasoning fields. Routing selection and automatic metadata remain separate + +### Deliberately maintained compatible offering + +```yaml +version: 1 +providers: + custom: + provider: openai + api_base: https://supplier.example/v1 + api_key: os.environ/CUSTOM_SUPPLIER_API_KEY +offerings: + - model_name: custom-flash + provider: custom + upstream_model: organization/model + source: manual + model_info: + mode: chat + context_window: 262144 + supports_function_calling: true + supports_parallel_function_calling: true + supports_reasoning: true + reasoning_effort_levels: [low, high, xhigh, max] + supported_endpoints: [/v1/chat/completions] + supported_modalities: [text, image] + supported_output_modalities: [text] + request_defaults: + output_token_budget: 65536 + output_token_budget_by_reasoning_effort: + max: 131072 +``` + +This is an explicit MANUAL use case, such as a compatible service without a usable inventory. The context and capability block are operator assertions requiring evidence, not supplier-discovered proof. There is deliberately no independent `max_output_tokens` capability. The output budgets do not establish one. Naming a connection `custom` or using this alias still invokes the generic `openai` adapter, not a custom model-family implementation + +## 6. A requested budget is not a capability + +The [budget helper](../../litellm/router_utils/model_request_defaults.py) defaults requests only for supported chat/Responses/Messages call types when a `request_defaults` policy exists and the caller has not supplied a non-null `max_tokens`, `max_completion_tokens` or `max_output_tokens` + +It chooses the budget's effort key from request effort first, then deployment request settings, then advertised `default_reasoning_effort`. It understands a direct effort string/object and `reasoning.effort`. An exact matching effort budget wins over the base output budget. If neither supplies a budget, nothing is injected. There is no guessed effort-strength conversion + +The helper returns `max_output_tokens` for Responses calls and `max_tokens` for the other supported calls. It does not inject a reasoning effort just because a default was advertised. An explicit caller budget wins over this configured default-budget policy, and conflicting deployment budget parameters are removed in that policy path. Native/provider request translation still determines the eventual wire format + +For the manual example, an omitted effort uses the base requested budget of `65536`; explicit `max` selects `131072`. Neither value means the physical output ceiling is known. This helper is defaulting, not a provider-independent quota or proof that a chosen budget is feasible + +An advertised reasoning default, a client electing to send that default, an explicit deployment request setting and a provider adapter's own default behavior are different things. The offering manager itself does not turn every advertised default into an outgoing request parameter. With no explicit effort setting or provider-specific injection, omission leaves the supplier's behavior in charge + +## 7. Publish one resolved generation, then authorize its catalog + +`ModelOfferingsManager._apply` compiles the resolved definitions through [Router.snapshot_with_model_list](../../litellm/router.py), setting `model_metadata_authoritative=True`. Native deployment records carry private typed provider policy, not an independent public policy registry + +The [serving view and ASGI middleware](../../litellm/proxy/offering_router.py) publish one generation atomically and pin that generation for each HTTP request, including the streaming response lifetime. Subsequent requests can see a newer generation while the earlier request finishes on its original view. This is a generation-pinning contract, not a claim that the whole native Router and all shared clients are deeply immutable. The facade blocks external model-definition mutation while reusing native resources + +The [shared model checks](../../litellm/proxy/auth/model_checks.py), native key/team/alias authorization and offering listing guard determine which selected public names the caller can see. Wildcard grants match live selected public Router names, not only old supplier registry entries. This does not broaden an exact key into access to other offerings. Alias lookup reuses the native listing/deployment context + +[create_model_info_response](../../litellm/proxy/utils.py) constructs the public object through the [discovery projection](../../litellm/proxy/model_discovery.py). In authoritative external mode it does not perform a second unrelated catalog fallback after offering resolution. The intended unpinned bundled fallback has already happened in `_deployment`. Private credentials, deployment identity and routing policy are not public model metadata + +## 8. Client options and the inference path + +A client reads its authorized gateway `/v1/models` result. A generic client can present the exact effort list as choices or variants, keep context/input/output distinct, show an advertised default and use a request-default policy when constructing a request. It must keep unknown fields unknown instead of manufacturing a supplier-specific limit or effort ladder + +Variant syntax and UI conversion belong to the client, not this LiteLLM PR. A client that rejects a requested effort variant when its catalog has no variants can do so before any inference request. This document does not claim to implement or test OpenCode's local validation or interactive UI, and no OpenCode source change is needed to define the gateway catalog contract + +For inference, the public alias resolves to a selected native deployment. The Router applies request-default budget policy and dispatches through the selected provider adapter. The [OpenRouter chat adapter](../../litellm/llms/openrouter/chat/transformation.py) preserves explicit gateway `max`/`xhigh`, and the [Messages bridge normalization](../../litellm/llms/anthropic/pass_through/utils.py) does not reduce OpenRouter effort using unrelated backend flags. Native Responses and Responses-to-chat paths retain their API-specific request shapes + +The offering guard merges the exact configured `only`/`allow_fallbacks: false` body and checks it again at deployment dispatch. Caller connection, deployment, provider, fallback, tier and parameter-dropping overrides cannot replace it. Pinned response-cache reads/writes are disabled to prevent output from another endpoint satisfying the request. No `require_parameters` setting or fallback supplier is silently added. Unsupported pinned API surfaces and native batches are rejected rather than bypassing selection + +## 9. Refresh, failure and present limitations + +Configuration polling defaults to five seconds, configurable from one to 300. Inventory polling defaults to 300 seconds, configurable from one to 86400. Known connection/base/credential or native-account changes invalidate the relevant prior state; changed inventory intervals or selected pinned-model requirements trigger fresh discovery. This is periodic discovery and atomic publication, not an instantaneous supplier query for every client request + +Inventory HTTP requests have a five-second timeout, a two-megabyte response bound and redirects disabled. The manager bounds concurrent provider refreshes to eight; endpoint requests within a provider are sequential. Cache identity includes supplier/base/authentication, and endpoint retention additionally checks credential scope. Positive compatible/endpoint inventories use a 300-second cache TTL and compatible/endpoint failures a 60-second negative TTL; the manager's forced scheduled refresh bypasses cached results + +A successful missing-model inventory removes an AUTO model. A failed fetch is not an empty inventory and retains last-known data only where identity permits. A successful empty endpoint inventory clears derived endpoint metadata without itself disabling an otherwise selected model. An initial endpoint failure leaves fields unknown rather than using unrestricted capacity. An invalid file edit or rejected candidate Router leaves the previous complete serving configuration in place + +Important boundaries of the current implementation remain + +- Automatic metadata is limited to fields the adapters recognize and the applicable bundled fallback can justify. It is not independent benchmarking, and unknown values can legitimately remain absent +- Raw Codex UI windows are retained internally without a generic physical-window guarantee. The Ultra default fallback is a normalization rule, not evidence of backend equivalence +- The [native ChatGPT Responses adapter](../../litellm/llms/chatgpt/responses/transformation.py) filters outgoing keys and currently excludes output-token budget fields. Do not promise that a configured budget is enforced on that backend merely because the shared helper can select it +- Pinning supports the current strict OpenRouter chat policy schema. Unknown routing controls, endpoint health/account-specific eligibility not represented by that schema, or an undocumented supplier-native effort mapping are not established by this metadata calculation +- An explicit override must be internally consistent and is not a deletion mechanism when set to null. A copied workaround can freeze AUTO fields until removed; it is not a required permanent policy +- Other clients may interpret catalog extensions differently. The regression evidence is in-process inventory, ASGI, Router and captured HTTP requests, not interactive client or paid supplier proof + +The [endpoint inventory tests](../../tests/unit/llms/openrouter/test_model_info.py), [offering catalog/request tests](../../tests/unit/proxy/test_model_offerings.py), [budget tests](../../tests/unit/router_utils/test_model_request_defaults.py) and [native account tests](../../tests/unit/llms/chatgpt/test_model_info.py) provide concrete examples of these contracts without deployment-local configuration or credentials diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index 62c54f42fd1..206d5b21ee4 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -1,5 +1,7 @@ # External model offerings +For the source-backed flow from supplier inventory to client options and requests, see [How selected offerings become a client catalog](ARCHITECTURE.md). It explains automatic metadata authority, intentional overrides, pinned endpoint guarantees and current adapter limitations + Set `general_settings.model_offerings_path` in the normal proxy configuration and keep its `model_list` empty. Disable `store_model_in_db`; the database may still serve authentication and usage tracking. Existing proxy configurations behave as before when the path is absent ```yaml @@ -61,7 +63,6 @@ The file is checked every five seconds by default, with a configurable range of Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The polling task stops at proxy shutdown after requests drain. Inventory refreshes reacquire the dedicated shared HTTP client through the native cache, whose eviction lifecycle owns client closure - Native subscription inventory uses the server-owned backend from `CHATGPT_API_BASE` / `OPENAI_CHATGPT_API_BASE`, or the provider default. An explicit connection URL must match that backend; per-model destinations cannot receive saved server OAuth credentials. Discovery never starts device login. Supplier-specific inventory behavior lives in the shared `litellm/llms/model_inventory.py` dispatcher. Connection identities use HMAC with the existing private server salt/master key. With that key configured, deployment IDs remain stable across restarts and change when supplier credentials or native account identity changes. No additional configured secret is required. Unconfigured standalone/development mode uses a private process key and does not promise ID stability across restart. Upgrading from earlier unkeyed offering IDs changes IDs once: encrypted reasoning tied to a removed ID may be discarded on replay or model switch, while readable history and model limits/defaults remain available. Native affinity callbacks use the request-pinned serving view during reloads. From 23ae9fbe177a21abc2b368f5e91df56516b42492 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 26 Oct 2026 08:21:38 +0100 Subject: [PATCH 31/34] fix(router): align metadata with native registration and reasoning paths --- examples/model_offerings/ARCHITECTURE.md | 8 +- examples/model_offerings/README.md | 2 +- litellm/llms/anthropic/pass_through/utils.py | 4 +- litellm/proxy/proxy_server.py | 2 +- litellm/router.py | 5 +- .../test_model_prices_and_context_window.py | 22 +-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 152 ++++++++++++++++++ 7 files changed, 166 insertions(+), 29 deletions(-) diff --git a/examples/model_offerings/ARCHITECTURE.md b/examples/model_offerings/ARCHITECTURE.md index c40d0ad6022..9ef9c05bd0d 100644 --- a/examples/model_offerings/ARCHITECTURE.md +++ b/examples/model_offerings/ARCHITECTURE.md @@ -4,7 +4,7 @@ Automatic supplier metadata is the normal path for AUTO offerings. Selecting an The schema still requires an explicit `source`. Normal automatic usage selects `source: auto` without copying a capability card; it does not mean an omitted `source` field is accepted -This walkthrough is verified against runtime commit `ec1e681d95dca0e6660e21fba9060ec9e3c7afde`. Relative source links follow the code in the same checkout. The feature extends native LiteLLM adapters, types, authentication, listing and Router paths; referencing those paths does not mean every implementation in them originated in this PR. See the [setup guide](README.md) for configuration and lifecycle instructions +This walkthrough includes the upstream sync through `dddbaa803d6e9fcc862ea7a5412f771cb1ae040e`. Relative source links follow the code in the same checkout, and the PR records validation against its published head. The feature extends native LiteLLM adapters, types, authentication, listing and Router paths; referencing those paths does not mean every implementation in them originated in this PR. See the [setup guide](README.md) for configuration and lifecycle instructions ```text supplier connection -> inventory -> native normalization -> offering resolution @@ -177,6 +177,8 @@ It chooses the budget's effort key from request effort first, then deployment re The helper returns `max_output_tokens` for Responses calls and `max_tokens` for the other supported calls. It does not inject a reasoning effort just because a default was advertised. An explicit caller budget wins over this configured default-budget policy, and conflicting deployment budget parameters are removed in that policy path. Native/provider request translation still determines the eventual wire format +The selector reads `reasoning_effort` and `reasoning.effort`; it is not a second implementation of every provider's thinking controls. In particular, its budget choice precedes later native request-level effort rounding. Do not assume an effort-specific default budget is recomputed for a different, unsupported effort that upstream subsequently rounds, or that every `output_config`/thinking-budget shape supplies this selector's effort key + For the manual example, an omitted effort uses the base requested budget of `65536`; explicit `max` selects `131072`. Neither value means the physical output ceiling is known. This helper is defaulting, not a provider-independent quota or proof that a chosen budget is feasible An advertised reasoning default, a client electing to send that default, an explicit deployment request setting and a provider adapter's own default behavior are different things. The offering manager itself does not turn every advertised default into an outgoing request parameter. With no explicit effort setting or provider-specific injection, omission leaves the supplier's behavior in charge @@ -199,6 +201,10 @@ Variant syntax and UI conversion belong to the client, not this LiteLLM PR. A cl For inference, the public alias resolves to a selected native deployment. The Router applies request-default budget policy and dispatches through the selected provider adapter. The [OpenRouter chat adapter](../../litellm/llms/openrouter/chat/transformation.py) preserves explicit gateway `max`/`xhigh`, and the [Messages bridge normalization](../../litellm/llms/anthropic/pass_through/utils.py) does not reduce OpenRouter effort using unrelated backend flags. Native Responses and Responses-to-chat paths retain their API-specific request shapes +Upstream's [deployment reasoning resolver](../../litellm/router_utils/reasoning_effort_capability.py) now owns deployment-declared effort handling across chat, Responses and Messages. An accepted declared value is kept. A ranked request outside the declared set can be rounded according to native up/down policy, or dropped under native `drop_params` behavior; explicit `default` resolves to omission. Cost-map-only choices do not activate deployment rounding. Supplier choices resolved into an AUTO deployment's own `model_info` do count as its declaration. The external offering schema does not add a parallel rounding framework or expose every native Router setting + +Consequently, preserving accepted OpenRouter gateway efforts is not an all-provider or all-input passthrough guarantee. The fork still avoids the provider adapter's old unconditional max-to-xhigh rename and legacy undeclared OpenRouter approximation, while reusing upstream's declared-effort resolver. Other provider adapters keep their native request semantics. The shared native [deployment-pricing resolver](../../litellm/router_utils/deployment_pricing.py) also owns base-model key precedence for listing; the former fork-only key lookup is removed + The offering guard merges the exact configured `only`/`allow_fallbacks: false` body and checks it again at deployment dispatch. Caller connection, deployment, provider, fallback, tier and parameter-dropping overrides cannot replace it. Pinned response-cache reads/writes are disabled to prevent output from another endpoint satisfying the request. No `require_parameters` setting or fallback supplier is silently added. Unsupported pinned API surfaces and native batches are rejected rather than bypassing selection ## 9. Refresh, failure and present limitations diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md index 206d5b21ee4..e32c77b7fb5 100644 --- a/examples/model_offerings/README.md +++ b/examples/model_offerings/README.md @@ -102,7 +102,7 @@ AUTO still checks model membership. Pinned AUTO offerings additionally discover Reasoning choices and defaults come from OpenRouter's model-level reasoning contract and are advertised only when every eligible endpoint declares the effort control. An endpoint's more restrictive declared choices narrow that set. This describes OpenRouter's accepted gateway options, not an independently discovered backend-native effort enum. An explicit empty choice list remains empty. Explicit null expands only to OpenRouter's documented gateway efforts, while omission does not declare choices. Mandatory reasoning excludes the disable value, and a disabled-by-default model does not claim an active default effort. Boolean reasoning support alone never supplies choices. Endpoint capability guarantees are combined conservatively, without using status, uptime or performance to change routing or availability -Every explicit `model_info` field remains authoritative. Existing handwritten context/output limits continue overriding discovery until the operator removes them. MANUAL offerings remain entirely handwritten. Requests without an effort leave the supplier default unchanged; explicit gateway effort values, including `max` and `xhigh`, are forwarded without backend-flag approximation on chat and translated Messages paths +Every explicit `model_info` field remains authoritative. Existing handwritten context/output limits continue overriding discovery until the operator removes them. MANUAL offerings remain entirely handwritten. An advertised default does not itself inject request effort. Accepted declared OpenRouter gateway efforts, including `max` and `xhigh`, are preserved without the old provider rename or backend-flag approximation. Upstream's native deployment resolver owns unsupported-value rounding/default/drop behavior across providers and routes; this is not a promise to forward every arbitrary effort unchanged Successful empty endpoint inventories clear derived endpoint metadata. Failed refreshes retain last-known endpoint metadata only for the same connection identity; an initial failure leaves unknown fields omitted, never replaced with another supplier's limits. Endpoint discovery does not disable an otherwise selected model or broaden its policy. Metadata refreshes publish atomically, and requests already running retain their previous snapshot. Policy edits still change the affected offering's private deployment ID diff --git a/litellm/llms/anthropic/pass_through/utils.py b/litellm/llms/anthropic/pass_through/utils.py index dcaa8e3f6b9..f466d1d3078 100644 --- a/litellm/llms/anthropic/pass_through/utils.py +++ b/litellm/llms/anthropic/pass_through/utils.py @@ -210,8 +210,6 @@ def normalize_reasoning_effort_value( handled where the thinking block is built. A deployment accepting no tier at all keeps the chain's floor, which is what every deployment degraded to before there was anything to ask. """ - if custom_llm_provider == "openrouter" or custom_llm_provider is None and model.startswith("openrouter/"): - return effort chain: Final = _EFFORT_DEGRADATION_CHAIN.get(effort) if chain is None: return effort @@ -223,6 +221,8 @@ def normalize_reasoning_effort_value( if deployment_rounds_reasoning_effort(deployment_model_info): return effort + if custom_llm_provider == "openrouter" or custom_llm_provider is None and model.startswith("openrouter/"): + return effort from litellm.utils import get_model_info try: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4bbb14fc0df..2d8d6c1115a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -167,8 +167,8 @@ from litellm.proxy.common_utils.token_count_settings import ( resolve_token_count_settings, ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog -from litellm.proxy.model_discovery import discover_model_metadata from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields +from litellm.proxy.model_discovery import discover_model_metadata from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, diff --git a/litellm/router.py b/litellm/router.py index 0f83dcc9bcc..35283da6485 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -290,7 +290,6 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache -from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults from litellm.router_utils.kubernetes_pod_discovery import ( KubernetesPodDiscovery, async_resolve_pods_after, @@ -298,6 +297,7 @@ from litellm.router_utils.kubernetes_pod_discovery import ( resolve_pods_after, resolve_pods_after_bound, ) +from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, ContinuationOutput, @@ -451,7 +451,6 @@ from litellm.utils import ( lookup_model_info_for_registration, model_info_with_pricing_overrides, provider_rejectable_params, - resolve_api_key_secret_reference, set_live_deployment_replay, ) @@ -11541,7 +11540,7 @@ class Router: deployment alone, which is what lets a refresh rebuild the same entries. """ model_info: Final = dict( - _MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True)) + _MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(mode="json", exclude_none=True)) ) for field in CustomPricingLiteLLMParams.model_fields: field_value = deployment.litellm_params.get(field) diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index ce9847dca33..13e9bd38a5c 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -736,27 +736,6 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): ] -def test_schema_and_model_info_preserve_independent_total_context(committed_schema, monkeypatch): - entry = { - "litellm_provider": "openai", - "context_window": 1200, - "max_input_tokens": 1100, - "max_output_tokens": 200, - "max_tokens": 200, - } - # The trusted-base generated catalog schema permits optional unknown properties. - # Runtime metadata supplies and validates the separate total-context contract. - assert build_validator(committed_schema).is_valid({"fixture-model": entry}) - monkeypatch.setattr(litellm, "model_cost", {"fixture-model": entry}) - info = litellm.get_model_info("fixture-model") - assert (info["context_window"], info["max_input_tokens"], info["max_output_tokens"], info["max_tokens"]) == ( - 1200, - 1100, - 200, - 200, - ) - - def test_native_model_info_and_lookup_preserve_reasoning_capabilities_and_request_defaults( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -766,6 +745,7 @@ def test_native_model_info_and_lookup_preserve_reasoning_capabilities_and_reques "context_window": 1200, "max_input_tokens": 1100, "max_output_tokens": 200, + "max_tokens": 200, "supports_function_calling": True, "supports_parallel_function_calling": False, "supports_reasoning": True, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 262e8b6deae..451bb9600e7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30205,6 +30205,11 @@ export interface components { * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called. */ model_list_healthy_only?: boolean | null; + /** + * Model Offerings Path + * @description Path to the authoritative external offering file, reloaded without restarting the proxy + */ + model_offerings_path?: string | null; /** * Otel * @description [BETA] OpenTelemetry support - this might change, use with caution. @@ -31799,6 +31804,8 @@ export interface components { model_info: components["schemas"]["litellm__types__router__ModelInfo"]; /** Model Name */ model_name: string; + /** @description Gateway-owned supplier selection, excluded from public model metadata */ + provider_policy?: components["schemas"]["OpenRouterProviderPolicy"] | null; } & { [key: string]: unknown; }; @@ -38600,6 +38607,21 @@ export interface components { /** Model Name */ model_name: string; }; + /** ModelRequestDefaults */ + ModelRequestDefaults: { + /** + * Output Token Budget + * @description Default requested output budget, not a physical model limit + */ + output_token_budget?: number | null; + /** + * Output Token Budget By Reasoning Effort + * @description Requested output budgets keyed by exact reasoning effort + */ + output_token_budget_by_reasoning_effort?: { + [key: string]: number; + } | null; + }; /** ModelResponse */ ModelResponse: { /** Choices */ @@ -39867,6 +39889,22 @@ export interface components { */ type: "openIdConnect"; }; + /** OpenRouterProviderPolicy */ + OpenRouterProviderPolicy: { + /** + * Allow Fallbacks + * @default false + * @constant + */ + allow_fallbacks: false; + /** Only */ + only: string[]; + /** + * Provider + * @constant + */ + provider: "openrouter"; + }; /** OpenSourceClassifierConfig */ OpenSourceClassifierConfig: { /** @@ -50210,6 +50248,16 @@ export interface components { litellm__proxy___types__ModelInfo: { /** Base Model */ base_model: ("gpt-4-1106-preview" | "gpt-4-32k" | "gpt-4" | "gpt-3.5-turbo-16k" | "gpt-3.5-turbo" | "text-embedding-ada-002") | null; + /** + * Context Window + * @description Maximum combined input and generated output tokens + */ + context_window?: number | null; + /** + * Default Reasoning Effort + * @description Effort used when the request omits it + */ + default_reasoning_effort?: string | null; /** Discoverable */ discoverable?: boolean | null; /** Id */ @@ -50219,6 +50267,16 @@ export interface components { * @default 0 */ input_cost_per_token: number | null; + /** + * Max Input Tokens + * @description Independent maximum input tokens + */ + max_input_tokens?: number | null; + /** + * Max Output Tokens + * @description Independent maximum generated output tokens + */ + max_output_tokens?: number | null; /** * Max Tokens * @default 2048 @@ -50231,6 +50289,43 @@ export interface components { * @default 0 */ output_cost_per_token: number | null; + /** + * Reasoning Effort Levels + * @description Exact effort values this route accepts + */ + reasoning_effort_levels?: string[] | null; + /** @description Configured request defaults, independent of supplier capability limits */ + request_defaults?: components["schemas"]["ModelRequestDefaults"] | null; + /** + * Supported Endpoints + * @description Inference endpoints exposed by this route + */ + supported_endpoints?: string[] | null; + /** + * Supported Modalities + * @description Accepted input modalities + */ + supported_modalities?: string[] | null; + /** + * Supported Output Modalities + * @description Supported output modalities + */ + supported_output_modalities?: string[] | null; + /** + * Supports Function Calling + * @description Whether this model accepts tool calls + */ + supports_function_calling?: boolean | null; + /** + * Supports Parallel Function Calling + * @description Whether this model accepts parallel tool calls + */ + supports_parallel_function_calling?: boolean | null; + /** + * Supports Reasoning + * @description Whether this model supports reasoning controls + */ + supports_reasoning?: boolean | null; } & { [key: string]: unknown; }; @@ -50281,6 +50376,11 @@ export interface components { cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; + /** + * Context Window + * @description Maximum combined input and generated output tokens + */ + context_window?: number | null; /** Cost Per Ptu Per Hour */ cost_per_ptu_per_hour?: number | null; /** Created At */ @@ -50292,6 +50392,11 @@ export interface components { * @default false */ db_model: boolean; + /** + * Default Reasoning Effort + * @description Effort used when the request omits it + */ + default_reasoning_effort?: string | null; /** Discoverable */ discoverable?: boolean | null; /** Enable Tag Filtering */ @@ -50308,6 +50413,16 @@ export interface components { internal_router_model?: boolean | null; /** Max Embedding Batch Size */ max_embedding_batch_size?: number | null; + /** + * Max Input Tokens + * @description Independent maximum input tokens + */ + max_input_tokens?: number | null; + /** + * Max Output Tokens + * @description Independent maximum generated output tokens + */ + max_output_tokens?: number | null; /** * Member Auto Router * @default false @@ -50329,6 +50444,43 @@ export interface components { ptu_shares?: { [key: string]: number; } | null; + /** + * Reasoning Effort Levels + * @description Exact effort values this route accepts + */ + reasoning_effort_levels?: string[] | null; + /** @description Configured request defaults, independent of supplier capability limits */ + request_defaults?: components["schemas"]["ModelRequestDefaults"] | null; + /** + * Supported Endpoints + * @description Inference endpoints exposed by this route + */ + supported_endpoints?: string[] | null; + /** + * Supported Modalities + * @description Accepted input modalities + */ + supported_modalities?: string[] | null; + /** + * Supported Output Modalities + * @description Supported output modalities + */ + supported_output_modalities?: string[] | null; + /** + * Supports Function Calling + * @description Whether this model accepts tool calls + */ + supports_function_calling?: boolean | null; + /** + * Supports Parallel Function Calling + * @description Whether this model accepts parallel tool calls + */ + supports_parallel_function_calling?: boolean | null; + /** + * Supports Reasoning + * @description Whether this model supports reasoning controls + */ + supports_reasoning?: boolean | null; /** Team Id */ team_id?: string | null; /** Team Public Model Name */ From 4d01e188ede4bcee8946644f221a6c88b4d61e0a Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 26 Oct 2026 09:28:07 +0100 Subject: [PATCH 32/34] fix: remediate CI dependencies and bound full-history checkout --- .github/workflows/test-unit.yml | 2 +- pyproject.toml | 2 +- uv.lock | 8 ++++---- vscode-extension/package-lock.json | 18 ++++++++++++++---- vscode-extension/package.json | 5 +++++ 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2eeea2d01b7..7180d3c6b7c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -951,7 +951,7 @@ jobs: dist: loadscope steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - timeout-minutes: 3 + timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 8 || 3 }} with: persist-credentials: false fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }} diff --git a/pyproject.toml b/pyproject.toml index 817c50df90b..95d9a277229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<3.20.4", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dotenv>=1.0.0,<2.0", diff --git a/uv.lock b/uv.lock index 531082730bb..f2b621d6f8d 100644 --- a/uv.lock +++ b/uv.lock @@ -2126,11 +2126,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.32.7" +version = "3.20.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, + { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] [[package]] @@ -4928,7 +4928,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, - { name = "filelock", specifier = ">=3.16.1,<4.0" }, + { name = "filelock", specifier = ">=3.16.1,<3.20.4" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 81e9425ef4e..16df05b41a2 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -189,19 +189,29 @@ } }, "node_modules/@azure/msal-node": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-6.0.1.tgz", - "integrity": "sha512-ixSO1Y/kCVRthRs+hSx/5qkwaunX1/RAePhlMN0wIpIQ4WEZ6AREGGnGd1AP0qspHVsAwzWQKGubpjh50JyJIQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-7.1.1.tgz", + "integrity": "sha512-zHXTJB38XVrsNaXrPBCMqEUjeyQysFfgmKtUHRDsEKTRjNIPckJ/w6fE5wBXrmJXBim1D3C7O7LOag0msPTrgw==", "dev": true, "license": "MIT", "dependencies": { - "@azure/msal-common": "16.14.1", + "@azure/msal-common": "16.14.3", "jsonwebtoken": "^9.0.0" }, "engines": { "node": ">=20" } }, + "node_modules/@azure/msal-node/node_modules/@azure/msal-common": { + "version": "16.14.3", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.14.3.tgz", + "integrity": "sha512-tz3GvMrR9V/plb2lDdW8qaO8CPpfZevdExBKjltzzbAtyhK0eB2UIvji1aU+gkyHj3fxQ2ie+LNHyMXdmo8wOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", diff --git a/vscode-extension/package.json b/vscode-extension/package.json index a3cf2dc7a5f..0fbe7ad0da2 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -83,5 +83,10 @@ "esbuild": "^0.28.2", "typescript": "^5.9.3", "vitest": "^4.1.12" + }, + "overrides": { + "@azure/identity": { + "@azure/msal-node": "^7.1.0" + } } } From 87ce3d94e4a55031c926d2efc9750b2968f4d2f9 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 26 Oct 2026 09:43:42 +0100 Subject: [PATCH 33/34] fix: align core distribution filelock security bound --- packaging/litellm-core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 890fa0fd773..ef1b868ee32 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "fastuuid>=0.14.0,<1.0", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<3.20.4", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dateutil>=2.8.2,<3.0", From 3f78bfabd79073673c74c9b26cd1b89e26c1c9b4 Mon Sep 17 00:00:00 2001 From: "J.H.S. Schlepers" Date: Mon, 26 Oct 2026 10:02:00 +0100 Subject: [PATCH 34/34] test: isolate AgentOps exporter HTTP collection by session --- .../integrations/otel/test_otel_v2_presets.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/unit/integrations/otel/test_otel_v2_presets.py b/tests/unit/integrations/otel/test_otel_v2_presets.py index 570c27a24e1..877ac071e23 100644 --- a/tests/unit/integrations/otel/test_otel_v2_presets.py +++ b/tests/unit/integrations/otel/test_otel_v2_presets.py @@ -93,13 +93,14 @@ def _agentops_exporter(): def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch(): pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - with respx.mock() as auth, requests_mock.Mocker() as collector: + with respx.mock() as auth: token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"}) - collector.post(_AGENTOPS_ENDPOINT, status_code=200) exporter = _agentops_exporter() assert token_route.call_count == 0, "building the exporter must not fetch the JWT" - results = [exporter.export(()), exporter.export(())] + with requests_mock.Mocker(session=exporter._agentops_session) as collector: + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + results = [exporter.export(()), exporter.export(())] assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS] assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}] @@ -111,11 +112,13 @@ def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_ def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails(): pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - with respx.mock() as auth, requests_mock.Mocker() as collector: + with respx.mock() as auth: auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down")) - collector.post(_AGENTOPS_ENDPOINT, status_code=200) + exporter = _agentops_exporter() + with requests_mock.Mocker(session=exporter._agentops_session) as collector: + collector.post(_AGENTOPS_ENDPOINT, status_code=200) - result = _agentops_exporter().export(()) + result = exporter.export(()) assert result is SpanExportResult.SUCCESS assert [request.headers.get("Authorization") for request in collector.request_history] == [None]