diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 42c73729a2b..ef41a2017ae 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3264,6 +3264,10 @@ def _check_model_access_helper( return True +# Read-only stand-in for an absent model_group_alias map. +_EMPTY_ALIAS_MAP: Final[Mapping[str, object]] = MappingProxyType({}) + + def _can_object_call_model( model: str | list[str], llm_router: Router | None, @@ -3331,7 +3335,7 @@ def _can_object_call_model( # requested name being in the allowlist grants nothing. This prevents a key # allowed ["stale-deployment-name"] from gaining access to a different # deployment via a canonical rewrite. - if llm_router and model not in (llm_router.model_group_alias or {}): + if llm_router and model not in (llm_router.model_group_alias or _EMPTY_ALIAS_MAP): canonical_target: Final[object] = llm_router.resolve_canonical_model_name( model=model, request_team_id=team_id, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index da75dc9d506..507006d4c91 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -226,6 +226,52 @@ def get_team_id_from_data(data: dict) -> str | None: return None +def _requested_model_metadata( + requested_model: str, +) -> dict: # mutable-ok: request metadata is consumed downstream as a plain dict + """A fresh metadata dict recording the client's original model spelling. + + Downstream logging consumes request metadata as a plain mutable dict, so + this is deliberately not frozen. + """ + return {"requested_model": requested_model} # mutable-ok: request metadata is consumed downstream as a plain dict + + +async def _canonical_target_is_allowed( + canonical_target: str, + llm_router: LitellmRouter, + user_api_key_dict: UserAPIKeyAuth | None, +) -> bool: + """Whether the caller may call a canonically-resolved target model group. + + AND-on-target: the caller must be permitted to call the *resolved* group. + The requested spelling having passed the earlier auth check is not enough -- + without this, a key whose allowlist holds only a stale, unserved name would + ride the rewrite onto a deployment it was never granted. (Auth ran on the + requested string before routing, when the target group was not yet known.) + + A denial returns False rather than raising, so the request falls through to + the same 400 an unresolvable model gets today and the response reveals + nothing about the target's existence. + """ + if user_api_key_dict is None: + return True + from litellm.proxy.auth.auth_checks import ( + can_key_call_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed + ) + + try: + await can_key_call_model( + model=canonical_target, + llm_model_list=llm_router.get_model_list(), + valid_token=user_api_key_dict, + llm_router=llm_router, + ) + except Exception: # noqa: BLE001 # any auth failure declines the rewrite; never widens access + return False + return True + + _shared_session_lock: asyncio.Lock | None = None @@ -687,31 +733,24 @@ async def route_request( # routing; the target group was not visible to it then.) On denial # the rewrite is simply declined -- the request falls through to # the same 400 it gets today, revealing nothing about the target. - target_allowed = True - if user_api_key_dict is not None: - from litellm.proxy.auth.auth_checks import ( - can_key_call_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed - ) - - try: - await can_key_call_model( - model=canonical_target, - llm_model_list=llm_router.get_model_list(), - valid_token=user_api_key_dict, - llm_router=llm_router, - ) - except Exception: # noqa: BLE001 # any auth failure declines the rewrite; never widens access - target_allowed = False + target_allowed: Final = await _canonical_target_is_allowed( + canonical_target=canonical_target, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + ) if target_allowed: # Preserve the client's spelling for spend logs / debugging -- # after the rewrite it is otherwise invisible downstream. metadata_field: Final = "litellm_metadata" if "litellm_metadata" in data else "metadata" - existing_metadata: object = data.get(metadata_field) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict + existing_metadata: Final[object] = data.get(metadata_field) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict + # `data` is rewritten in place here because every other route in + # this function does the same; see the rebind-ok notes below. if isinstance(existing_metadata, dict): existing_metadata.setdefault("requested_model", requested_model) # pyright: ignore[reportUnknownMemberType] - metadata dict is untyped else: - data[metadata_field] = {"requested_model": requested_model} - data["model"] = canonical_target + stamp: Final = _requested_model_metadata(requested_model) + data[metadata_field] = stamp # rebind-ok: in-place rewrite, as everywhere in route_request + data["model"] = canonical_target # rebind-ok: in-place data rewrite, as everywhere in route_request return getattr(llm_router, f"{route_type}")(**data) # if no route found then it's a bad request diff --git a/litellm/router.py b/litellm/router.py index aee52e50f7b..3b0e6eb64a4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -636,7 +636,7 @@ class Router: # Lazily-built (provider, canonical_name) -> model_group index for # ``resolve_canonical_model_name``. Invalidated alongside the model # group info cache and on cost-map mutation (generation counter). - self._canonical_model_index: dict[tuple[str, str], str | None] | None = None + self._canonical_model_index: Mapping[tuple[str, str], str | None] | None = None self._canonical_model_index_cost_generation: int = -1 # (requested, target) pairs already announced at INFO, so a hot path # logs once per distinct pair rather than once per request. Keyed on @@ -645,7 +645,8 @@ class Router: # the signal used to size demand for possible follow-up resolution # rules, so a second spelling silently sharing the first's log line # would undercount it. - self._canonical_resolution_logged: set[tuple[str, str]] = set() + # mutable-ok: grow-only log-dedup ledger, intentionally accumulates across requests + self._canonical_resolution_logged: set[tuple[str, str]] = set() # mutable-ok: grow-only log-dedup ledger self._init_routing_groups(None) self.deployment_affinity_ttl_seconds = deployment_affinity_ttl_seconds @@ -10711,7 +10712,7 @@ class Router: """ return resolve_model_group_alias(self.model_group_alias, model) - def _get_canonical_model_index(self) -> dict[tuple[str, str], str | None]: + def _get_canonical_model_index(self) -> Mapping[tuple[str, str], str | None]: """The ``(provider, canonical_name) -> model group`` index, built on demand. Rebuilt when the model list changes (the index is dropped by @@ -10721,10 +10722,10 @@ class Router: cost_generation: Final = get_model_cost_mutation_generation() if self._canonical_model_index is None or self._canonical_model_index_cost_generation != cost_generation: try: - self._canonical_model_index = build_canonical_index(cast("list[DeploymentTypedDict]", self.model_list)) + self._canonical_model_index = build_canonical_index(self.model_list) except Exception as exc: # noqa: BLE001 # index construction must never brick a router; degrade to 'strict' verbose_router_logger.error("canonical-resolution: index build failed, disabling feature: %s", exc) - self._canonical_model_index = {} + self._canonical_model_index = MappingProxyType({}) self._canonical_model_index_cost_generation = cost_generation return self._canonical_model_index diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py index 82344910bc9..71a8a4add94 100644 --- a/litellm/router_utils/canonical_model_resolution.py +++ b/litellm/router_utils/canonical_model_resolution.py @@ -35,12 +35,12 @@ succeeds today can never be re-pointed by this module. See also ``Router.resolve_canonical_model_name``. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import Final import litellm from litellm._logging import verbose_router_logger -from litellm.types.router import DeploymentTypedDict # Cost-map fields that must match exactly for two names to be called the same # model. Pricing equality is a tripwire against false identity, not the @@ -59,10 +59,13 @@ _IDENTITY_ATTESTING_FIELDS: Final[tuple[str, ...]] = ( # billing path the operator never sanctioned. _AMBIGUOUS: Final = None +# Read-only stand-in for a missing sub-mapping on a deployment row. +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) + def _cost_map_entry(model: str) -> Mapping[str, object] | None: """The cost-map entry for ``model``, or None when absent.""" - entry = litellm.model_cost.get(model) + entry: Final = litellm.model_cost.get(model) return entry if isinstance(entry, Mapping) else None @@ -138,8 +141,8 @@ def _undated_variants(model: str) -> tuple[str, ...]: def build_canonical_index( - deployments: list[DeploymentTypedDict], -) -> dict[tuple[str, str], str | None]: + deployments: Sequence[Mapping[str, object]], +) -> Mapping[tuple[str, str], str | None]: """Map ``(provider, canonical_name) -> model group`` for ``deployments``. A model group is indexed only when every one of its deployments agrees on @@ -160,25 +163,24 @@ def build_canonical_index( Never raises: a malformed deployment or cost-map entry degrades to a smaller index, never to a router that fails to boot. """ - index: dict[tuple[str, str], str | None] = {} - group_identity: dict[str, tuple[str, str] | None] = {} + # Both accumulate across the deployment scan, then the result is frozen into + # a MappingProxyType before it leaves this function. + index: Final[dict[tuple[str, str], str | None]] = {} # mutable-ok: local accumulator, frozen on return + group_identity: Final[dict[str, tuple[str, str] | None]] = {} # mutable-ok: local accumulator, never escapes for deployment in deployments: try: - model_info = deployment.get("model_info") or {} - if isinstance( # pyright: ignore[reportUnnecessaryIsInstance] - config/DB rows can violate the TypedDict - model_info, Mapping - ) and model_info.get("team_id"): + model_info = deployment.get("model_info") or _EMPTY # rebind-ok: per-deployment loop variable + if isinstance(model_info, Mapping) and model_info.get("team_id"): continue - # ``model_name``/``model`` are typed Required[str], but this index is - # built from operator config and DB rows that can violate the type, - # so both are validated at runtime rather than trusted. - model_group: object = deployment.get("model_name") - litellm_params = deployment.get("litellm_params") or {} - underlying: object = litellm_params.get("model") - if not isinstance(model_group, str) or not isinstance( # pyright: ignore[reportUnnecessaryIsInstance] - config/DB rows can violate the TypedDict - underlying, str - ): + # Deployments come from operator config and DB rows, so the declared + # str types are validated at runtime rather than trusted. + model_group = deployment.get("model_name") # rebind-ok: per-deployment loop variable + litellm_params = deployment.get("litellm_params") or _EMPTY # rebind-ok: per-deployment loop variable + underlying = ( + litellm_params.get("model") if isinstance(litellm_params, Mapping) else None + ) # rebind-ok: per-deployment loop variable + if not isinstance(model_group, str) or not isinstance(underlying, str): continue identity = canonicalize(underlying) @@ -197,18 +199,24 @@ def build_canonical_index( continue provider, canonical_name = identity # Index the canonical spelling plus any dated<->undated sibling the cost - # map attests is the same model. - spellings: list[str] = [canonical_name] - spellings.extend( + # map attests is the same model: the undated form of a dated canonical + # name, and every dated cost-map entry whose undated form is this one. + undated_siblings = tuple( # rebind-ok: per-group loop variable candidate for candidate in _undated_variants(canonical_name) if _same_model_per_cost_map(canonical_name, candidate) ) - for dated, entry in litellm.model_cost.items(): - if not isinstance(dated, str) or not isinstance(entry, Mapping) or dated in spellings: - continue - if _undated_variants(dated) == (canonical_name,) and _same_model_per_cost_map(canonical_name, dated): - spellings.append(dated) + dated_siblings = tuple( # rebind-ok: per-group loop variable + dated + for dated, entry in litellm.model_cost.items() + if isinstance(dated, str) + and isinstance(entry, Mapping) + and dated != canonical_name + and dated not in undated_siblings + and _undated_variants(dated) == (canonical_name,) + and _same_model_per_cost_map(canonical_name, dated) + ) + spellings = (canonical_name, *undated_siblings, *dated_siblings) # rebind-ok: per-group loop variable for spelling in spellings: key = (provider, spelling) @@ -228,7 +236,7 @@ def build_canonical_index( model_group, ) - return index + return MappingProxyType(index) def lookup(