From 55c499100e1c7d3cecb4c769d0a3ffaa29d92ff2 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur Date: Sat, 15 Aug 2026 08:13:19 -0700 Subject: [PATCH 1/8] feat: same-provider canonical model-name resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Feature A: when a request names a model using one spelling (e.g. dated 'claude-haiku-4-5-20251001') that the gateway serves under a different spelling (e.g. 'anthropic/claude-haiku-4-5'), route the request to the serving deployment instead of failing with a 403/400. Resolution only fires after all existing routes (exact name, alias, wildcard, default_deployment, team routes) have declined, so a request that succeeds today is never re-pointed (I1). Identity is enforced strictly: same provider (never cross-provider like Bedrock to Vertex), same model per cost-map attestation (never family/version hops like claude-sonnet-4-5 to claude-sonnet-5) (I2). Auth is AND-on-target: the caller must be allowed to call the resolved target group; the requested spelling's presence in an allowlist alone grants nothing, preventing privilege escalation via stale spellings (I3). Observability: requested name preserved in request metadata, resolution logged at INFO per (requested, target) pair, Prometheus counter for cardinality. Changes: - litellm/router_utils/canonical_model_resolution.py: new module with canonicalize, build index, lookup functions. - litellm/router.py: Router.resolve_canonical_model_name(), index caching + invalidation, config flag model_name_resolution. - litellm/types/router.py: RouterGeneralSettings.model_name_resolution field. - litellm/proxy/route_llm_request.py: resolve hook before 400, re-auth on target, metadata stamping. - litellm/proxy/auth/auth_checks.py: canonical lookup in _can_object_call_model with AND-on-target semantics. - tests/test_litellm/router_utils/test_canonical_model_resolution.py: 23 property tests covering I1–I4, cross-provider block, ambiguity fail-closed, auth semantics. Defaults to 'canonical' (on); users can set model_name_resolution: strict in router_settings to opt out entirely. Fixes the Claude Code case: dated Haiku requests now resolve across harness versions without client-side env vars. Follow-up: once operator data shows a single provider deployment of all Anthropic models, Feature B (cross-deployment resolution when only 1 provider exists) becomes safe to implement under similar guards. Co-Authored-By: Claude --- litellm/proxy/auth/auth_checks.py | 22 ++ litellm/proxy/route_llm_request.py | 46 +++ litellm/router.py | 73 +++++ .../canonical_model_resolution.py | 225 ++++++++++++++ litellm/types/router.py | 5 + .../test_canonical_model_resolution.py | 289 ++++++++++++++++++ 6 files changed, 660 insertions(+) create mode 100644 litellm/router_utils/canonical_model_resolution.py create mode 100644 tests/test_litellm/router_utils/test_canonical_model_resolution.py diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 3d8fed18423..d80232bbab3 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3323,6 +3323,28 @@ def _can_object_call_model( ): return True + # Canonical resolution: if the requested model has no manual alias but + # *provably* serves the same model as something in the request's + # allowed-models list, that's a match. Crucially, this is NOT OR-semantics + # (as with manual aliases, where the admin blessed the alias itself). It's + # AND-on-target: the target must be explicitly allowed, and the raw + # requested name being in the allowlist grants nothing. This prevents a key + # allowed ["stale-deployment-name"] from gaining access to a different + # deployment via a canonical rewrite. + if llm_router and model not in (llm_router.model_group_alias or {}): + canonical_target: Final = llm_router.resolve_canonical_model_name( + model=model, + request_team_id=team_id, + ) + if canonical_target is not None and _check_model_access_helper( + model=canonical_target, + llm_router=llm_router, + models=models, + team_model_aliases=team_model_aliases, + team_id=team_id, + ): + return True + raise ProxyException( message=f"{object_type} not allowed to access model. This {object_type} can only access models={models}. Tried to access {model}", type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type=object_type), diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index b347360a939..14e0a5aba9c 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -663,6 +663,52 @@ async def route_request( elif user_model is not None or route_type == "allm_passthrough_route": return getattr(litellm, f"{route_type}")(**data) + # Last resort before failing: the requested name may be a different spelling + # of a model this router already serves (e.g. a harness sending the dated + # 'claude-haiku-4-5-20251001' at a gateway that deploys it as + # 'anthropic/claude-haiku-4-5'). Resolution is same-provider and + # identity-attested only, and runs here -- after every configured route, + # including wildcards and default_deployment, has declined -- so it can only + # turn a hard failure into a success, never re-point working traffic. + if llm_router is not None and isinstance(data.get("model"), str): + canonical_target: Final = llm_router.resolve_canonical_model_name( + model=data["model"], + request_team_id=team_id, + ) + if canonical_target is not None: + # AND-on-target: the caller must be allowed to call the *resolved* + # group. The requested spelling passing the earlier auth check is + # not enough -- without this, a key whose allowlist holds only a + # stale unserved name would ride the rewrite onto a deployment it + # was never granted. (Auth ran on the requested string before + # routing; the target group was not visible to it then.) On denial + # the rewrite is simply declined -- the request falls through to + # the same 400 it gets today, revealing nothing about the target. + target_allowed = True + if user_api_key_dict is not None: + from litellm.proxy.auth.auth_checks import can_key_call_model + + try: + await can_key_call_model( + model=canonical_target, + llm_model_list=llm_router.get_model_list(), + valid_token=user_api_key_dict, + llm_router=llm_router, + ) + except Exception: + target_allowed = False + if target_allowed: + # Preserve the client's spelling for spend logs / debugging -- + # after the rewrite it is otherwise invisible downstream. + metadata_field: Final = "litellm_metadata" if "litellm_metadata" in data else "metadata" + existing_metadata = data.get(metadata_field) + if isinstance(existing_metadata, dict): + existing_metadata.setdefault("requested_model", data["model"]) + else: + data[metadata_field] = {"requested_model": data["model"]} + data["model"] = canonical_target + return getattr(llm_router, f"{route_type}")(**data) + # if no route found then it's a bad request route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError( diff --git a/litellm/router.py b/litellm/router.py index 4a450cf3c7c..40700b16291 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -109,6 +109,12 @@ from litellm.router_utils.clientside_credential_handler import ( get_dynamic_litellm_params, is_clientside_credential, ) +from litellm.router_utils.canonical_model_resolution import ( + build_canonical_index, +) +from litellm.router_utils.canonical_model_resolution import ( + lookup as canonical_lookup, +) from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, @@ -218,6 +224,7 @@ from litellm.utils import ( Rules, function_setup, get_llm_provider, + get_model_cost_mutation_generation, get_non_default_completion_params, get_secret, get_utc_datetime, @@ -626,6 +633,14 @@ class Router: # ``litellm.proxy.auth.auth_checks._is_model_cost_zero``. self._zero_cost_cache: dict[str, bool] = {} self._routing_group_rows: tuple[DeploymentTypedDict, ...] | None = None + # Lazily-built (provider, canonical_name) -> model_group index for + # ``resolve_canonical_model_name``. Invalidated alongside the model + # group info cache and on cost-map mutation (generation counter). + self._canonical_model_index: dict[tuple[str, str], str | None] | None = None + self._canonical_model_index_cost_generation: int = -1 + # Targets already announced at INFO, so a hot path logs once per target + # rather than once per request. + self._canonical_resolution_logged: set[str] = set() self._init_routing_groups(None) self.deployment_affinity_ttl_seconds = deployment_affinity_ttl_seconds @@ -10233,6 +10248,7 @@ class Router: self._cached_get_model_group_info.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None + self._canonical_model_index = None def _invalidate_access_groups_cache(self) -> None: """Invalidate the cached access groups. @@ -10690,6 +10706,63 @@ class Router: """ return resolve_model_group_alias(self.model_group_alias, model) + def _get_canonical_model_index(self) -> dict[tuple[str, str], str | None]: + """The ``(provider, canonical_name) -> model group`` index, built on demand. + + Rebuilt when the model list changes (the index is dropped by + ``_invalidate_model_group_info_cache``) or when ``litellm.model_cost`` + mutates, since identity attestation reads it. + """ + cost_generation: Final = get_model_cost_mutation_generation() + if self._canonical_model_index is None or self._canonical_model_index_cost_generation != cost_generation: + try: + self._canonical_model_index = build_canonical_index(self.model_list) + except Exception as exc: + # Never let index construction brick a router: degrade to + # 'strict' behaviour instead. + verbose_router_logger.error("canonical-resolution: index build failed, disabling feature: %s", exc) + self._canonical_model_index = {} + self._canonical_model_index_cost_generation = cost_generation + return self._canonical_model_index + + def resolve_canonical_model_name(self, model: str, request_team_id: str | None = None) -> str | None: + """The model group that provably serves ``model`` under another spelling. + + Returns None unless every one of these holds: + - ``router_general_settings.model_name_resolution`` is ``"canonical"`` + - ``model`` is not already served (``is_recognized_model``) + - no team route, pattern/wildcard route, or ``default_deployment`` would + take the request -- those are operator-configured catch-alls and must + keep winning + - a single model group matches ``model``'s canonical identity *on the + same provider* + + Callers must treat a non-None result as authorization-relevant: the + target model group is what the key/team must be permitted to call. + """ + if self.router_general_settings.model_name_resolution != "canonical": + return None + if not model or self.is_recognized_model(model): + return None + # Operator-configured catch-alls outrank inference. + if self.default_deployment is not None or len(self.pattern_router.patterns) > 0: + return None + if request_team_id is not None and request_team_id in self.team_pattern_routers: + return None + if model in self.deployment_names: + return None + + target: Final = canonical_lookup(self._get_canonical_model_index(), model) + if target is None: + return None + # A target whose deployments have all been removed is not a live route. + if not self.model_name_to_deployment_indices.get(target): + return None + if target not in self._canonical_resolution_logged: + self._canonical_resolution_logged.add(target) + verbose_router_logger.info("canonical-resolution: '%s' -> '%s'", model, target) + return target + def _get_deployment_by_litellm_model(self, model: str) -> list: """ Get the deployment by litellm model. diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py new file mode 100644 index 00000000000..114ccfaa15c --- /dev/null +++ b/litellm/router_utils/canonical_model_resolution.py @@ -0,0 +1,225 @@ +"""Same-provider canonical model-name resolution. + +Harnesses hardcode concrete model IDs. A client that asks for +``claude-haiku-4-5-20251001`` against a gateway that serves the very same model +under the deployment name ``anthropic/claude-haiku-4-5`` gets a 403/400 today, +even though the gateway *is* serving what was asked for. Only the spelling +differs. + +This module builds a ``canonical name -> model group`` index from the router's +own deployments so that spelling difference can be bridged. Two hard rules keep +the bridge from becoming a guess: + +1. **Identity, not similarity.** Two names are equivalent only when the model + cost map attests they are the same model: both present, same + ``litellm_provider``, same ``mode``, and identical pricing. Family/version + hops (``claude-sonnet-4-5`` -> ``claude-sonnet-5``) are never equivalences, + because that changes which model answers. +2. **Never across providers.** The requested name's inferred provider must equal + the target deployment's provider. The same weights on Bedrock, Vertex, and + the first-party API differ in credentials, data residency, quota pool, and + billing; picking between them is an operator decision, not something a + gateway should infer. Cross-provider mapping stays available through an + explicit ``model_group_alias``. + +Resolution is a last resort: callers consult it only after every existing route +(exact name, deployment id, ``model_group_alias``, routing group, team route, +wildcard/pattern route, ``default_deployment``) has declined, so a request that +succeeds today can never be re-pointed by this module. + +See also ``Router.resolve_canonical_model_name``. +""" + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +import litellm +from litellm._logging import verbose_router_logger + +if TYPE_CHECKING: + from litellm.types.router import DeploymentTypedDict + +# Cost-map fields that must match exactly for two names to be called the same +# model. Pricing equality is a tripwire against false identity, not the +# definition of it -- the provider/mode checks below carry that weight. +_IDENTITY_ATTESTING_FIELDS: Final[tuple[str, ...]] = ( + "litellm_provider", + "mode", + "input_cost_per_token", + "output_cost_per_token", + "max_input_tokens", + "max_output_tokens", +) + +# Sentinel stored in the index when two distinct model groups claim the same +# canonical identity. Resolution then declines rather than silently picking a +# billing path the operator never sanctioned. +_AMBIGUOUS: Final = None + + +def _cost_map_entry(model: str) -> Mapping[str, object] | None: + """The cost-map entry for ``model``, or None when absent.""" + entry: Final = litellm.model_cost.get(model) + return entry if isinstance(entry, Mapping) else None + + +def _same_model_per_cost_map(name_a: str, name_b: str) -> bool: + """Whether the cost map attests ``name_a`` and ``name_b`` are one model. + + Both names must be present with identical provider, mode, and pricing. A + customer fine-tune or an unknown vanity name is absent from the map and so + can never be equated with anything -- which is the point. + """ + entry_a: Final = _cost_map_entry(name_a) + entry_b: Final = _cost_map_entry(name_b) + if entry_a is None or entry_b is None: + return False + return all(entry_a.get(field) == entry_b.get(field) for field in _IDENTITY_ATTESTING_FIELDS) + + +def _infer_provider(model: str) -> str | None: + """The provider LiteLLM would route ``model`` to, or None if undecidable. + + Wraps ``get_llm_provider``, which raises ``BadRequestError`` for names it + cannot place. An undecidable name simply never participates in resolution. + """ + try: + _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) + except Exception: + return None + return custom_llm_provider or None + + +def canonicalize(model: str) -> tuple[str, str] | None: + """Reduce ``model`` to a ``(provider, canonical_name)`` identity. + + The canonical name is the model string with any LiteLLM provider-route + prefix removed (``anthropic/claude-opus-5`` -> ``claude-opus-5``), which is + LiteLLM's own routing syntax rather than part of the model's identity. The + provider is carried alongside so equality checks are always provider-scoped. + + Returns None when the provider cannot be inferred. + """ + if not model: + return None + provider: Final = _infer_provider(model) + if provider is None: + return None + # get_llm_provider returns the model with its routing prefix stripped, which + # is exactly the normalization wanted here. + try: + stripped, _, _, _ = litellm.get_llm_provider(model=model) + except Exception: + return None + return (provider, stripped or model) + + +def _undated_variants(model: str) -> tuple[str, ...]: + """Plausible dated<->undated spellings of ``model``, unvalidated. + + Purely syntactic candidate generation: every candidate is still gated by + ``_same_model_per_cost_map`` before it is treated as an equivalence, so a + coincidental date-like suffix on an unrelated model cannot create a false + match (it will not be in the cost map, or will not match on pricing). + + Only an 8-digit ``-YYYYMMDD`` suffix is considered. Deliberately narrower + than the cost-lookup heuristics in ``litellm.utils`` (which strip any + trailing ``-\\d+`` and would conflate ``gemini-1.5-pro-001`` with ``-002``): + a wrong match there mis-prices a log line, a wrong match here serves the + wrong model. + """ + parts: Final = model.rsplit("-", 1) + if len(parts) == 2 and len(parts[1]) == 8 and parts[1].isdigit(): + return (parts[0],) + return () + + +def build_canonical_index( + deployments: list["DeploymentTypedDict"], +) -> dict[tuple[str, str], str | None]: + """Map ``(provider, canonical_name) -> model group`` for ``deployments``. + + A model group is indexed only when every one of its deployments agrees on + the same canonical identity; mixed groups are skipped. When two groups claim + one identity the entry is set to ``_AMBIGUOUS`` (None) so lookups decline. + + Never raises: a malformed deployment or cost-map entry degrades to a smaller + index, never to a router that fails to boot. + """ + index: dict[tuple[str, str], str | None] = {} + group_identity: dict[str, tuple[str, str] | None] = {} + + for deployment in deployments: + try: + model_group = deployment.get("model_name") + litellm_params = deployment.get("litellm_params") or {} + underlying = litellm_params.get("model") if isinstance(litellm_params, Mapping) else None + if not isinstance(model_group, str) or not isinstance(underlying, str): + continue + + identity = canonicalize(underlying) + if model_group in group_identity and group_identity[model_group] != identity: + # Deployments in this group disagree about what they serve; the + # group cannot stand for a single canonical identity. + group_identity[model_group] = None + continue + group_identity.setdefault(model_group, identity) + except Exception as exc: # pragma: no cover - defensive + verbose_router_logger.debug("canonical-resolution: skipping deployment: %s", exc) + continue + + for model_group, identity in group_identity.items(): + if identity is None: + continue + provider, canonical_name = identity + # Index the canonical spelling plus any dated<->undated sibling the cost + # map attests is the same model. + spellings: list[str] = [canonical_name] + for candidate in _undated_variants(canonical_name): + if _same_model_per_cost_map(canonical_name, candidate): + spellings.append(candidate) + for dated, entry in litellm.model_cost.items(): + if not isinstance(entry, Mapping) or dated in spellings: + continue + if _undated_variants(dated) == (canonical_name,) and _same_model_per_cost_map(canonical_name, dated): + spellings.append(dated) + + for spelling in spellings: + key = (provider, spelling) + existing = index.get(key, "__absent__") + if existing == "__absent__": + index[key] = model_group + elif existing != model_group: + # Two groups, same identity: decline rather than choose. + index[key] = _AMBIGUOUS + verbose_router_logger.info( + "canonical-resolution: '%s' is served by more than one model group " + "(%s, %s); auto-resolution disabled for it. Add an explicit " + "model_group_alias to pick one.", + spelling, + existing, + model_group, + ) + + return index + + +def lookup( + index: Mapping[tuple[str, str], str | None], + requested_model: str, +) -> str | None: + """The model group serving ``requested_model``, or None. + + None covers every decline: unknown provider, no identity match, or an + ambiguous identity. Pure dict lookup after canonicalization -- no I/O. + """ + identity: Final = canonicalize(requested_model) + if identity is None: + return None + target: Final = index.get(identity) + if target is None: + return None + # A request already naming its serving group is not a rewrite. + if target == requested_model: + return None + return target diff --git a/litellm/types/router.py b/litellm/types/router.py index 217364c48b7..94d7930781e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -759,6 +759,11 @@ class RouterGeneralSettings(BaseModel): pass_through_all_models: bool = Field( default=False ) # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding + model_name_resolution: Literal["canonical", "strict"] = Field( + default="canonical" + ) # "canonical": an unknown requested model name that provably names an already-served model + # (same provider, cost-map-attested identity, e.g. dated vs undated spelling) is routed to the + # serving model group instead of failing. "strict": unknown names fail exactly as before. class RouterRateLimitErrorBasic(ValueError): diff --git a/tests/test_litellm/router_utils/test_canonical_model_resolution.py b/tests/test_litellm/router_utils/test_canonical_model_resolution.py new file mode 100644 index 00000000000..0e282af8516 --- /dev/null +++ b/tests/test_litellm/router_utils/test_canonical_model_resolution.py @@ -0,0 +1,289 @@ +"""Tests for same-provider canonical model-name resolution. + +Invariants under test (see litellm/router_utils/canonical_model_resolution.py): +- I1: resolution never re-points a request that any existing route accepts + (exact name, alias, wildcard/pattern, default_deployment, team routes). +- I2: identity only -- same provider, cost-map-attested; never a family/version + hop, never cross-provider (Bedrock/Vertex/first-party are distinct). +- I3: auth is AND-on-target -- the resolved group must itself be allowed. +- Ambiguity fails closed. +""" + +import pytest + +from litellm import Router +from litellm.proxy.auth.auth_checks import _can_object_call_model +from litellm.proxy._types import ProxyException +from litellm.router_utils.canonical_model_resolution import ( + build_canonical_index, + canonicalize, + lookup, +) + +ANTHROPIC_GROUP = "anthropic/claude-haiku-4-5" +DATED = "claude-haiku-4-5-20251001" +UNDATED = "claude-haiku-4-5" +BEDROCK_FORM = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +@pytest.fixture +def anthropic_router() -> Router: + return Router( + model_list=[ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-test"}, + } + ] + ) + + +class TestCanonicalize: + def test_strips_provider_route_prefix(self): + assert canonicalize("anthropic/claude-haiku-4-5") == ("anthropic", UNDATED) + + def test_bare_dated_name_infers_provider(self): + assert canonicalize(DATED) == ("anthropic", DATED) + + def test_bedrock_form_is_bedrock_not_anthropic(self): + identity = canonicalize(BEDROCK_FORM) + assert identity is not None + assert identity[0] == "bedrock" + + def test_unknown_model_returns_none(self): + assert canonicalize("totally-made-up-model-xyz") is None + + def test_empty_returns_none(self): + assert canonicalize("") is None + + +class TestBuildIndexAndLookup: + def test_dated_and_undated_spellings_resolve(self): + index = build_canonical_index( + [ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + } + ] + ) + assert lookup(index, DATED) == ANTHROPIC_GROUP + assert lookup(index, UNDATED) == ANTHROPIC_GROUP + + def test_no_family_version_hop(self): + """claude-sonnet-5 must never resolve to a haiku group (different model).""" + index = build_canonical_index( + [ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + } + ] + ) + assert lookup(index, "claude-sonnet-5") is None + + def test_cross_provider_never_matches(self): + """An Anthropic-form request must not land on a Bedrock-only deployment, + and a Bedrock-form request must not land on a first-party deployment.""" + bedrock_only = build_canonical_index( + [ + { + "model_name": "claude-haiku-bedrock", + "litellm_params": {"model": f"bedrock/{BEDROCK_FORM}"}, + } + ] + ) + assert lookup(bedrock_only, DATED) is None + assert lookup(bedrock_only, UNDATED) is None + # Same-provider spelling still works for the Bedrock group. + assert lookup(bedrock_only, BEDROCK_FORM) == "claude-haiku-bedrock" + + anthropic_only = build_canonical_index( + [ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + } + ] + ) + assert lookup(anthropic_only, BEDROCK_FORM) is None + + def test_vertex_deployment_does_not_capture_anthropic_request(self): + index = build_canonical_index( + [ + { + "model_name": "claude-haiku-vertex", + "litellm_params": {"model": "vertex_ai/claude-haiku-4-5"}, + } + ] + ) + assert lookup(index, DATED) is None + + def test_ambiguous_identity_fails_closed(self): + """Two same-provider groups serving one model: resolution must decline.""" + index = build_canonical_index( + [ + { + "model_name": "haiku-prod", + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + }, + { + "model_name": "haiku-experiments", + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + }, + ] + ) + assert lookup(index, DATED) is None + assert lookup(index, UNDATED) is None + + def test_request_for_own_group_name_is_not_a_rewrite(self): + index = build_canonical_index( + [ + { + "model_name": UNDATED, # group named exactly the canonical name + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + } + ] + ) + assert lookup(index, UNDATED) is None + + def test_malformed_deployments_are_skipped_not_fatal(self): + index = build_canonical_index( + [ + {"model_name": None, "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "ok-group", "litellm_params": None}, + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + }, + ] + ) + assert lookup(index, DATED) == ANTHROPIC_GROUP + + +class TestRouterResolveCanonicalModelName: + def test_claude_code_case(self, anthropic_router: Router): + assert anthropic_router.resolve_canonical_model_name(DATED) == ANTHROPIC_GROUP + + def test_recognized_model_short_circuits(self, anthropic_router: Router): + """I1: a name the router already serves is never rewritten.""" + assert anthropic_router.resolve_canonical_model_name(ANTHROPIC_GROUP) is None + + def test_strict_mode_disables_resolution(self): + from litellm.types.router import RouterGeneralSettings + + router = Router( + model_list=[ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-test"}, + } + ], + router_general_settings=RouterGeneralSettings(model_name_resolution="strict"), + ) + assert router.resolve_canonical_model_name(DATED) is None + + def test_wildcard_route_outranks_resolution(self): + """I1: an operator catch-all keeps winning; resolution declines entirely.""" + router = Router( + model_list=[ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-test"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-test"}, + }, + ] + ) + assert router.resolve_canonical_model_name(DATED) is None + + def test_default_deployment_outranks_resolution(self): + router = Router( + model_list=[ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-test"}, + }, + { + "model_name": "*", + "litellm_params": {"model": "*", "api_key": "sk-test"}, + }, + ] + ) + assert router.resolve_canonical_model_name(DATED) is None + + def test_manual_alias_outranks_resolution(self): + """A model_group_alias for the same spelling wins (is_recognized_model).""" + router = Router( + model_list=[ + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-test"}, + }, + { + "model_name": "other-group", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "sk-test"}, + }, + ], + model_group_alias={DATED: "other-group"}, + ) + assert router.resolve_canonical_model_name(DATED) is None + + def test_index_rebuilds_after_model_list_change(self, anthropic_router: Router): + assert anthropic_router.resolve_canonical_model_name(DATED) == ANTHROPIC_GROUP + anthropic_router.set_model_list( + [ + { + "model_name": "gpt-group", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}, + } + ] + ) + assert anthropic_router.resolve_canonical_model_name(DATED) is None + + +class TestAuthAndOnTarget: + """I3: an auto-resolved request is allowed iff the TARGET is allowed.""" + + def test_target_allowed_grants_requested_spelling(self, anthropic_router: Router): + assert ( + _can_object_call_model( + model=DATED, + llm_router=anthropic_router, + models=[ANTHROPIC_GROUP], + object_type="key", + ) + is True + ) + + def test_target_not_allowed_denies(self, anthropic_router: Router): + with pytest.raises(ProxyException): + _can_object_call_model( + model=DATED, + llm_router=anthropic_router, + models=["some-other-model"], + object_type="key", + ) + + def test_unrelated_model_still_denied(self, anthropic_router: Router): + with pytest.raises(ProxyException): + _can_object_call_model( + model="claude-sonnet-5", + llm_router=anthropic_router, + models=[ANTHROPIC_GROUP], + object_type="key", + ) + + def test_unrestricted_key_unchanged(self, anthropic_router: Router): + # Empty allowlist = unrestricted; behavior must not change. + assert ( + _can_object_call_model( + model=DATED, + llm_router=anthropic_router, + models=[], + object_type="key", + ) + is True + ) From d952e5e87e2f0c265ae18c496189b184830f5661 Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 08:36:36 -0700 Subject: [PATCH 2/8] chore: type-discipline cleanup for canonical model resolution basedpyright: new module at 0 errors; modified files back to baseline (+1 structural 'Return type is Any' on the new route return, matching the file's existing pattern on every route return). ruff clean. Co-Authored-By: Claude --- litellm/proxy/route_llm_request.py | 15 ++++++---- litellm/router.py | 14 ++++++---- .../canonical_model_resolution.py | 28 ++++++++++++------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 14e0a5aba9c..00500a45360 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -670,9 +670,10 @@ async def route_request( # identity-attested only, and runs here -- after every configured route, # including wildcards and default_deployment, has declined -- so it can only # turn a hard failure into a success, never re-point working traffic. - if llm_router is not None and isinstance(data.get("model"), str): + requested_model: Final[object] = data.get("model") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict + if llm_router is not None and isinstance(requested_model, str): canonical_target: Final = llm_router.resolve_canonical_model_name( - model=data["model"], + model=requested_model, request_team_id=team_id, ) if canonical_target is not None: @@ -686,7 +687,9 @@ async def route_request( # the same 400 it gets today, revealing nothing about the target. target_allowed = True if user_api_key_dict is not None: - from litellm.proxy.auth.auth_checks import can_key_call_model + from litellm.proxy.auth.auth_checks import ( + can_key_call_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed + ) try: await can_key_call_model( @@ -701,11 +704,11 @@ async def route_request( # Preserve the client's spelling for spend logs / debugging -- # after the rewrite it is otherwise invisible downstream. metadata_field: Final = "litellm_metadata" if "litellm_metadata" in data else "metadata" - existing_metadata = data.get(metadata_field) + existing_metadata: object = data.get(metadata_field) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict if isinstance(existing_metadata, dict): - existing_metadata.setdefault("requested_model", data["model"]) + existing_metadata.setdefault("requested_model", requested_model) # pyright: ignore[reportUnknownMemberType] - metadata dict is untyped else: - data[metadata_field] = {"requested_model": data["model"]} + data[metadata_field] = {"requested_model": requested_model} data["model"] = canonical_target return getattr(llm_router, f"{route_type}")(**data) diff --git a/litellm/router.py b/litellm/router.py index 40700b16291..2d4f7491803 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -104,17 +104,17 @@ from litellm.router_utils.batch_utils import ( replace_model_in_jsonl, should_replace_model_in_jsonl, ) -from litellm.router_utils.client_initalization_utils import InitalizeCachedClient -from litellm.router_utils.clientside_credential_handler import ( - get_dynamic_litellm_params, - is_clientside_credential, -) from litellm.router_utils.canonical_model_resolution import ( build_canonical_index, ) from litellm.router_utils.canonical_model_resolution import ( lookup as canonical_lookup, ) +from litellm.router_utils.client_initalization_utils import InitalizeCachedClient +from litellm.router_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, + is_clientside_credential, +) from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, @@ -10716,7 +10716,9 @@ class Router: cost_generation: Final = get_model_cost_mutation_generation() if self._canonical_model_index is None or self._canonical_model_index_cost_generation != cost_generation: try: - self._canonical_model_index = build_canonical_index(self.model_list) + self._canonical_model_index = build_canonical_index( + cast("list[DeploymentTypedDict]", self.model_list) + ) except Exception as exc: # Never let index construction brick a router: degrade to # 'strict' behaviour instead. diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py index 114ccfaa15c..e4e215d019c 100644 --- a/litellm/router_utils/canonical_model_resolution.py +++ b/litellm/router_utils/canonical_model_resolution.py @@ -1,3 +1,8 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportAny=false +# litellm.model_cost is loaded dynamically from a JSON price map (see +# litellm.litellm_core_utils.get_model_cost_map) and is untyped at that +# boundary, same as every other reader of it in litellm/utils.py. Every value +# pulled from it here is re-validated with isinstance before use. """Same-provider canonical model-name resolution. Harnesses hardcode concrete model IDs. A client that asks for @@ -31,13 +36,11 @@ See also ``Router.resolve_canonical_model_name``. """ from collections.abc import Mapping -from typing import TYPE_CHECKING, Final +from typing import Final import litellm from litellm._logging import verbose_router_logger - -if TYPE_CHECKING: - from litellm.types.router import DeploymentTypedDict +from litellm.types.router import DeploymentTypedDict # Cost-map fields that must match exactly for two names to be called the same # model. Pricing equality is a tripwire against false identity, not the @@ -59,7 +62,7 @@ _AMBIGUOUS: Final = None def _cost_map_entry(model: str) -> Mapping[str, object] | None: """The cost-map entry for ``model``, or None when absent.""" - entry: Final = litellm.model_cost.get(model) + entry = litellm.model_cost.get(model) return entry if isinstance(entry, Mapping) else None @@ -135,7 +138,7 @@ def _undated_variants(model: str) -> tuple[str, ...]: def build_canonical_index( - deployments: list["DeploymentTypedDict"], + deployments: list[DeploymentTypedDict], ) -> dict[tuple[str, str], str | None]: """Map ``(provider, canonical_name) -> model group`` for ``deployments``. @@ -151,10 +154,15 @@ def build_canonical_index( for deployment in deployments: try: - model_group = deployment.get("model_name") + # ``model_name``/``model`` are typed Required[str], but this index is + # built from operator config and DB rows that can violate the type, + # so both are validated at runtime rather than trusted. + model_group: object = deployment.get("model_name") litellm_params = deployment.get("litellm_params") or {} - underlying = litellm_params.get("model") if isinstance(litellm_params, Mapping) else None - if not isinstance(model_group, str) or not isinstance(underlying, str): + underlying: object = litellm_params.get("model") + if not isinstance(model_group, str) or not isinstance( # pyright: ignore[reportUnnecessaryIsInstance] - config/DB rows can violate the TypedDict + underlying, str + ): continue identity = canonicalize(underlying) @@ -179,7 +187,7 @@ def build_canonical_index( if _same_model_per_cost_map(canonical_name, candidate): spellings.append(candidate) for dated, entry in litellm.model_cost.items(): - if not isinstance(entry, Mapping) or dated in spellings: + if not isinstance(dated, str) or not isinstance(entry, Mapping) or dated in spellings: continue if _undated_variants(dated) == (canonical_name,) and _same_model_per_cost_map(canonical_name, dated): spellings.append(dated) From 6df125ff6d42b0455d5a431e33fbfc20925f12cd Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 11:09:14 -0700 Subject: [PATCH 3/8] fix: address CI failures on canonical model resolution Three CI gates failed on the initial push; all three were real: 1. lint (strict-rule budget): BLE001 +5, PERF401 +1 over base. The defensive 'except Exception' catches are intentional (an unplaceable name or a malformed deployment must never fail a request or brick a router), so they now carry '# noqa: BLE001' with justifications, matching the convention already used in router.py and common_request_processing.py. The candidate -spelling loop becomes a generator passed to list.extend (PERF401). Both rules are now back at base parity. 2. code-quality (router_code_coverage): the checker only scans test files whose filename contains 'router', so tests/.../test_canonical_model_resolution.py was invisible to it and both new Router methods read as untested. Renamed to test_router_canonical_model_resolution.py and added direct coverage for _get_canonical_model_index (memoization + fenced build failure). Checker now reports untested_perc: 0.0. 3. proxy-infra (test_route_non_a2a_model_raises_error_if_not_in_router): a real regression. The test drives route_request with a Mock() router, so resolve_canonical_model_name returned a truthy Mock and the hook treated it as a resolved target instead of raising. Both hooks (routing and auth) now require an actual non-empty str before acting on a resolution -- correct hardening independent of the test: a stub or partially-initialised router must never be read as a grant. Full affected suite: 517 passed. basedpyright: new module 0 errors. Co-Authored-By: Claude --- litellm/proxy/auth/auth_checks.py | 6 ++++-- litellm/proxy/route_llm_request.py | 8 ++++--- litellm/router.py | 4 +--- .../canonical_model_resolution.py | 14 +++++++------ ...test_router_canonical_model_resolution.py} | 21 +++++++++++++++++++ 5 files changed, 39 insertions(+), 14 deletions(-) rename tests/test_litellm/router_utils/{test_canonical_model_resolution.py => test_router_canonical_model_resolution.py} (90%) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index d80232bbab3..8a44a7bab33 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3332,11 +3332,13 @@ def _can_object_call_model( # allowed ["stale-deployment-name"] from gaining access to a different # deployment via a canonical rewrite. if llm_router and model not in (llm_router.model_group_alias or {}): - canonical_target: Final = llm_router.resolve_canonical_model_name( + canonical_target: Final[object] = llm_router.resolve_canonical_model_name( model=model, request_team_id=team_id, ) - if canonical_target is not None and _check_model_access_helper( + # Require a real model-group name; a non-string from a router stub must + # never be treated as a grant. + if isinstance(canonical_target, str) and canonical_target and _check_model_access_helper( model=canonical_target, llm_router=llm_router, models=models, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 00500a45360..da75dc9d506 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -672,11 +672,13 @@ async def route_request( # turn a hard failure into a success, never re-point working traffic. requested_model: Final[object] = data.get("model") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict if llm_router is not None and isinstance(requested_model, str): - canonical_target: Final = llm_router.resolve_canonical_model_name( + canonical_target: Final[object] = llm_router.resolve_canonical_model_name( model=requested_model, request_team_id=team_id, ) - if canonical_target is not None: + # Require a real model-group name: a router stub that returns a + # non-string (e.g. a test double) must not be read as "resolved". + if isinstance(canonical_target, str) and canonical_target: # AND-on-target: the caller must be allowed to call the *resolved* # group. The requested spelling passing the earlier auth check is # not enough -- without this, a key whose allowlist holds only a @@ -698,7 +700,7 @@ async def route_request( valid_token=user_api_key_dict, llm_router=llm_router, ) - except Exception: + except Exception: # noqa: BLE001 # any auth failure declines the rewrite; never widens access target_allowed = False if target_allowed: # Preserve the client's spelling for spend logs / debugging -- diff --git a/litellm/router.py b/litellm/router.py index 2d4f7491803..8f02a5255e1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10719,9 +10719,7 @@ class Router: self._canonical_model_index = build_canonical_index( cast("list[DeploymentTypedDict]", self.model_list) ) - except Exception as exc: - # Never let index construction brick a router: degrade to - # 'strict' behaviour instead. + except Exception as exc: # noqa: BLE001 # index construction must never brick a router; degrade to 'strict' verbose_router_logger.error("canonical-resolution: index build failed, disabling feature: %s", exc) self._canonical_model_index = {} self._canonical_model_index_cost_generation = cost_generation diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py index e4e215d019c..ebb4688aeb1 100644 --- a/litellm/router_utils/canonical_model_resolution.py +++ b/litellm/router_utils/canonical_model_resolution.py @@ -88,7 +88,7 @@ def _infer_provider(model: str) -> str | None: """ try: _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) - except Exception: + except Exception: # noqa: BLE001 # an unplaceable name simply never resolves; never fail the request return None return custom_llm_provider or None @@ -112,7 +112,7 @@ def canonicalize(model: str) -> tuple[str, str] | None: # is exactly the normalization wanted here. try: stripped, _, _, _ = litellm.get_llm_provider(model=model) - except Exception: + except Exception: # noqa: BLE001 # an unplaceable name simply never resolves; never fail the request return None return (provider, stripped or model) @@ -172,7 +172,7 @@ def build_canonical_index( group_identity[model_group] = None continue group_identity.setdefault(model_group, identity) - except Exception as exc: # pragma: no cover - defensive + except Exception as exc: # noqa: BLE001 # pragma: no cover - a malformed deployment must not abort the index build verbose_router_logger.debug("canonical-resolution: skipping deployment: %s", exc) continue @@ -183,9 +183,11 @@ def build_canonical_index( # Index the canonical spelling plus any dated<->undated sibling the cost # map attests is the same model. spellings: list[str] = [canonical_name] - for candidate in _undated_variants(canonical_name): - if _same_model_per_cost_map(canonical_name, candidate): - spellings.append(candidate) + spellings.extend( + candidate + for candidate in _undated_variants(canonical_name) + if _same_model_per_cost_map(canonical_name, candidate) + ) for dated, entry in litellm.model_cost.items(): if not isinstance(dated, str) or not isinstance(entry, Mapping) or dated in spellings: continue diff --git a/tests/test_litellm/router_utils/test_canonical_model_resolution.py b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py similarity index 90% rename from tests/test_litellm/router_utils/test_canonical_model_resolution.py rename to tests/test_litellm/router_utils/test_router_canonical_model_resolution.py index 0e282af8516..c9b06888312 100644 --- a/tests/test_litellm/router_utils/test_canonical_model_resolution.py +++ b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py @@ -231,6 +231,27 @@ class TestRouterResolveCanonicalModelName: ) assert router.resolve_canonical_model_name(DATED) is None + def test_get_canonical_model_index_builds_and_caches(self, anthropic_router: Router): + """The index is built on demand, memoized, and keyed by identity.""" + index = anthropic_router._get_canonical_model_index() + assert index[("anthropic", UNDATED)] == ANTHROPIC_GROUP + # Second call returns the same memoized object (no rebuild). + assert anthropic_router._get_canonical_model_index() is index + + def test_get_canonical_model_index_survives_build_failure( + self, anthropic_router: Router, monkeypatch: pytest.MonkeyPatch + ): + """A failing index build degrades to 'strict', never raises.""" + import litellm.router as router_module + + def boom(*_args: object, **_kwargs: object) -> dict: + raise RuntimeError("cost map exploded") + + monkeypatch.setattr(router_module, "build_canonical_index", boom) + anthropic_router._canonical_model_index = None + assert anthropic_router._get_canonical_model_index() == {} + assert anthropic_router.resolve_canonical_model_name(DATED) is None + def test_index_rebuilds_after_model_list_change(self, anthropic_router: Router): assert anthropic_router.resolve_canonical_model_name(DATED) == ANTHROPIC_GROUP anthropic_router.set_model_list( From 84e2d4e23cdc2cf5dd0c5b176827476553e08072 Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 11:18:57 -0700 Subject: [PATCH 4/8] fix: address review findings (team-scoping leak, sentinel collision, log dedup) Three issues flagged by automated PR review (greptile-apps, veria-ai), all confirmed real and reproduced before fixing: 1. [High/Security] Team-owned deployments entered the global canonical index unconditionally. A no-team (unrestricted) key could request an unclaimed spelling of a model (e.g. the dated Anthropic ID) whose only server was a team's private deployment, resolve onto it, and use that team's credentials/quota -- target-authorization passes for unrestricted keys and doesn't itself re-derive team ownership. Fixed by excluding any deployment with model_info.team_id set from the index entirely: a team boundary is an access/billing boundary exactly like the cross-provider boundary this module already respects, and team-scoped models remain reachable exactly as before, via team_public_model_name through the existing team-route machinery this module never touches. 2. [P1] Sentinel collision defeated the ambiguity guard: index.get(key, '__absent__') treated a model group literally named '__absent__' as a missing entry, so a second group with the same identity would silently overwrite it instead of triggering the ambiguity decline. Fixed with a proper 'in' check. 3. [P2] Log deduplication was keyed on target alone, so a second distinct requested spelling resolving to an already-logged target never got its own log line -- undercounting the (requested, target) cardinality that's the whole point of the observability story (sizing follow-up-resolution demand). Now keyed on the (requested, target) pair. Added 5 regression tests reproducing each bug pre-fix and asserting the fixed behavior. Full affected suite: 522 passed. router_code_coverage: 0.0% untested. ruff-strict BLE001/PERF401: unchanged at base parity. basedpyright: new module 0 errors. Co-Authored-By: Claude --- litellm/router.py | 16 ++- .../canonical_model_resolution.py | 23 +++- .../test_router_canonical_model_resolution.py | 102 ++++++++++++++++++ 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8f02a5255e1..94ca7854233 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -638,9 +638,14 @@ class Router: # group info cache and on cost-map mutation (generation counter). self._canonical_model_index: dict[tuple[str, str], str | None] | None = None self._canonical_model_index_cost_generation: int = -1 - # Targets already announced at INFO, so a hot path logs once per target - # rather than once per request. - self._canonical_resolution_logged: set[str] = set() + # (requested, target) pairs already announced at INFO, so a hot path + # logs once per distinct pair rather than once per request. Keyed on + # the pair, not just the target, so each new requested spelling that + # resolves to an already-seen target is still observable -- this is + # the signal used to size demand for possible follow-up resolution + # rules, so a second spelling silently sharing the first's log line + # would undercount it. + self._canonical_resolution_logged: set[tuple[str, str]] = set() self._init_routing_groups(None) self.deployment_affinity_ttl_seconds = deployment_affinity_ttl_seconds @@ -10758,8 +10763,9 @@ class Router: # A target whose deployments have all been removed is not a live route. if not self.model_name_to_deployment_indices.get(target): return None - if target not in self._canonical_resolution_logged: - self._canonical_resolution_logged.add(target) + log_key: Final = (model, target) + if log_key not in self._canonical_resolution_logged: + self._canonical_resolution_logged.add(log_key) verbose_router_logger.info("canonical-resolution: '%s' -> '%s'", model, target) return target diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py index ebb4688aeb1..82344910bc9 100644 --- a/litellm/router_utils/canonical_model_resolution.py +++ b/litellm/router_utils/canonical_model_resolution.py @@ -146,6 +146,17 @@ def build_canonical_index( the same canonical identity; mixed groups are skipped. When two groups claim one identity the entry is set to ``_AMBIGUOUS`` (None) so lookups decline. + Team-owned deployments (``model_info.team_id`` set) are never indexed. A + team boundary is an operator-drawn access/billing boundary exactly like a + provider boundary (see the module docstring's rule 2): auto-resolution must + not cross it. Concretely, without this exclusion a global (no-team) key + could request a team's deployment under an unclaimed spelling -- e.g. the + dated Anthropic ID -- and land on that team's credentials and quota, since + ``is_recognized_model``/target-authorization checks pass for unrestricted + keys and don't themselves re-derive team ownership. A team-scoped model + remains reachable exactly as it is today: by its team_public_model_name, + through the existing team-route machinery, which this module never touches. + Never raises: a malformed deployment or cost-map entry degrades to a smaller index, never to a router that fails to boot. """ @@ -154,6 +165,11 @@ def build_canonical_index( for deployment in deployments: try: + model_info = deployment.get("model_info") or {} + if isinstance( # pyright: ignore[reportUnnecessaryIsInstance] - config/DB rows can violate the TypedDict + model_info, Mapping + ) and model_info.get("team_id"): + continue # ``model_name``/``model`` are typed Required[str], but this index is # built from operator config and DB rows that can violate the type, # so both are validated at runtime rather than trusted. @@ -196,10 +212,11 @@ def build_canonical_index( for spelling in spellings: key = (provider, spelling) - existing = index.get(key, "__absent__") - if existing == "__absent__": + if key not in index: index[key] = model_group - elif existing != model_group: + continue + existing = index[key] + if existing != model_group: # Two groups, same identity: decline rather than choose. index[key] = _AMBIGUOUS verbose_router_logger.info( diff --git a/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py index c9b06888312..93109f56d04 100644 --- a/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py +++ b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py @@ -136,6 +136,70 @@ class TestBuildIndexAndLookup: assert lookup(index, DATED) is None assert lookup(index, UNDATED) is None + def test_ambiguity_guard_not_defeated_by_group_named_like_sentinel(self): + """Regression: the ambiguity check used to compare against the string + '__absent__' as an "is this key missing" sentinel. A model group + literally named '__absent__' collided with that sentinel and could + silently overwrite a same-identity entry instead of triggering the + ambiguity decline.""" + index = build_canonical_index( + [ + { + "model_name": "__absent__", + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + }, + { + "model_name": "haiku-2", + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + }, + ] + ) + assert lookup(index, DATED) is None + assert lookup(index, UNDATED) is None + + def test_team_owned_deployment_never_indexed(self): + """Regression: a team-owned deployment (model_info.team_id set) must + never enter the global canonical index. Without this, a no-team key + could request an unclaimed spelling of a model whose only server is a + team's private deployment and land on that team's credentials/quota -- + a team boundary is an access/billing boundary exactly like the + cross-provider boundary and must not be crossed by inference.""" + index = build_canonical_index( + [ + { + "model_name": "internal-team-model-xyz", + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + "model_info": { + "team_id": "team-A", + "team_public_model_name": "claude-haiku-4-5", + }, + }, + ] + ) + assert lookup(index, DATED) is None + assert lookup(index, UNDATED) is None + + def test_team_owned_deployment_does_not_block_global_sibling(self): + """A team-owned deployment coexisting with a global deployment of the + same identity must not suppress resolution to the global one.""" + index = build_canonical_index( + [ + { + "model_name": "internal-team-model-xyz", + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + "model_info": { + "team_id": "team-A", + "team_public_model_name": "claude-haiku-4-5", + }, + }, + { + "model_name": ANTHROPIC_GROUP, + "litellm_params": {"model": "anthropic/claude-haiku-4-5"}, + }, + ] + ) + assert lookup(index, DATED) == ANTHROPIC_GROUP + def test_request_for_own_group_name_is_not_a_rewrite(self): index = build_canonical_index( [ @@ -169,6 +233,44 @@ class TestRouterResolveCanonicalModelName: """I1: a name the router already serves is never rewritten.""" assert anthropic_router.resolve_canonical_model_name(ANTHROPIC_GROUP) is None + def test_no_team_caller_never_resolves_onto_team_owned_deployment(self): + """Regression: end-to-end version of the team-leak fix. A no-team + caller asking for an unclaimed spelling must not resolve onto a + deployment that is the sole server of that identity but is owned by a + team -- that would leak the team's credentials/quota to a global key.""" + router = Router( + model_list=[ + { + "model_name": "internal-team-model-xyz", + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "team-secret"}, + "model_info": {"team_id": "team-A", "team_public_model_name": "claude-haiku-4-5"}, + }, + ] + ) + assert router.resolve_canonical_model_name(DATED, request_team_id=None) is None + # Even the requesting team's own id must not resolve through this path -- + # team-scoped models are reached via the existing team-route machinery, + # not via canonical inference. + assert router.resolve_canonical_model_name(DATED, request_team_id="team-A") is None + + def test_log_dedup_keyed_on_pair_not_target_alone(self, anthropic_router: Router, caplog: pytest.LogCaptureFixture): + """Regression: a second distinct requested spelling resolving to an + already-logged target must still get its own log line -- the + (requested, target) cardinality is the signal used to size demand for + follow-up resolution rules, so deduping on target alone would + undercount it.""" + import logging + + with caplog.at_level(logging.INFO, logger="LiteLLM Router"): + assert anthropic_router.resolve_canonical_model_name(DATED) == ANTHROPIC_GROUP + assert anthropic_router.resolve_canonical_model_name(UNDATED) == ANTHROPIC_GROUP + # Re-requesting the same spelling must not double-log. + assert anthropic_router.resolve_canonical_model_name(DATED) == ANTHROPIC_GROUP + messages = [r.message for r in caplog.records if "canonical-resolution" in r.message] + assert any(DATED in m for m in messages) + assert any(UNDATED in m for m in messages) + assert sum(1 for m in messages if DATED in m) == 1 + def test_strict_mode_disables_resolution(self): from litellm.types.router import RouterGeneralSettings From 646bde17afb4732e585abe0c85564bf804edef8e Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 11:24:57 -0700 Subject: [PATCH 5/8] style: apply ruff format to canonical resolution changes The lint job's 'Check ruff format' step (separate from the strict-rule budget step that failed earlier) flagged two files. Both diffs are confined to lines added by this PR -- the multi-line isinstance/and condition in _can_object_call_model, and the build_canonical_index call in _get_canonical_model_index -- so no unrelated formatting is swept in. Cosmetic only, no semantic change. ruff format --check: clean. ruff check: clean. basedpyright (new module): 0 errors. strict-rule budgets BLE001/PERF401: unchanged at base parity. Tests: 32 passed. Co-Authored-By: Claude --- litellm/proxy/auth/auth_checks.py | 16 ++++++++++------ litellm/router.py | 4 +--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8a44a7bab33..42c73729a2b 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3338,12 +3338,16 @@ def _can_object_call_model( ) # Require a real model-group name; a non-string from a router stub must # never be treated as a grant. - if isinstance(canonical_target, str) and canonical_target and _check_model_access_helper( - model=canonical_target, - llm_router=llm_router, - models=models, - team_model_aliases=team_model_aliases, - team_id=team_id, + if ( + isinstance(canonical_target, str) + and canonical_target + and _check_model_access_helper( + model=canonical_target, + llm_router=llm_router, + models=models, + team_model_aliases=team_model_aliases, + team_id=team_id, + ) ): return True diff --git a/litellm/router.py b/litellm/router.py index 94ca7854233..aee52e50f7b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10721,9 +10721,7 @@ class Router: cost_generation: Final = get_model_cost_mutation_generation() if self._canonical_model_index is None or self._canonical_model_index_cost_generation != cost_generation: try: - self._canonical_model_index = build_canonical_index( - cast("list[DeploymentTypedDict]", self.model_list) - ) + self._canonical_model_index = build_canonical_index(cast("list[DeploymentTypedDict]", self.model_list)) except Exception as exc: # noqa: BLE001 # index construction must never brick a router; degrade to 'strict' verbose_router_logger.error("canonical-resolution: index build failed, disabling feature: %s", exc) self._canonical_model_index = {} From 5a677ca27115e18ddcea11cc5c0cd3d213f22d72 Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 11:51:50 -0700 Subject: [PATCH 6/8] style: conform to repo type-discipline rules (LIT001/002/006/010/011) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lint job has three independent gates; this is the third (type-discipline budget, delta vs base). Rather than keep discovering them one CI round at a time, ran scripts/check_type_discipline.py locally over the whole tree and brought every LIT rule back to exact base parity. Conformed rather than blanket-suppressed where the rule was right: - build_canonical_index takes a read-only Sequence[Mapping[str, object]] and returns a MappingProxyType, so the index cannot be mutated by callers after construction. This also removed the cast() at the router call site (LIT006) and the now-stale reportUnnecessaryIsInstance suppressions -- with object values the runtime validation is genuinely necessary, not redundant. - spellings is built functionally (two generator-fed tuples) instead of seed-then-append. - The AND-on-target auth re-check moved into a small _canonical_target_is_allowed helper, which removes a rebound flag and makes the AND-on-target contract documented in one place. - Genuine cases carry the house-style justification comments: mutable-ok for the grow-only log-dedup ledger and the local accumulators (frozen on return), rebind-ok for the in-place rewrites route_request already performs on every other route, and for per-iteration loop variables (where Final is illegal). Verified locally against every gate this time, not just the ones CI had surfaced: pytest 522 passed · ruff format clean · ruff check clean · ruff-strict BLE001/PERF401 at base parity (2957/22) · type-discipline all LIT rules at base parity · basedpyright new module 0 errors · router_code_coverage 0.0% untested. Co-Authored-By: Claude --- litellm/proxy/auth/auth_checks.py | 6 +- litellm/proxy/route_llm_request.py | 75 ++++++++++++++----- litellm/router.py | 11 +-- .../canonical_model_resolution.py | 66 +++++++++------- 4 files changed, 105 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 42c73729a2b..ef41a2017ae 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3264,6 +3264,10 @@ def _check_model_access_helper( return True +# Read-only stand-in for an absent model_group_alias map. +_EMPTY_ALIAS_MAP: Final[Mapping[str, object]] = MappingProxyType({}) + + def _can_object_call_model( model: str | list[str], llm_router: Router | None, @@ -3331,7 +3335,7 @@ def _can_object_call_model( # requested name being in the allowlist grants nothing. This prevents a key # allowed ["stale-deployment-name"] from gaining access to a different # deployment via a canonical rewrite. - if llm_router and model not in (llm_router.model_group_alias or {}): + if llm_router and model not in (llm_router.model_group_alias or _EMPTY_ALIAS_MAP): canonical_target: Final[object] = llm_router.resolve_canonical_model_name( model=model, request_team_id=team_id, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index da75dc9d506..507006d4c91 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -226,6 +226,52 @@ def get_team_id_from_data(data: dict) -> str | None: return None +def _requested_model_metadata( + requested_model: str, +) -> dict: # mutable-ok: request metadata is consumed downstream as a plain dict + """A fresh metadata dict recording the client's original model spelling. + + Downstream logging consumes request metadata as a plain mutable dict, so + this is deliberately not frozen. + """ + return {"requested_model": requested_model} # mutable-ok: request metadata is consumed downstream as a plain dict + + +async def _canonical_target_is_allowed( + canonical_target: str, + llm_router: LitellmRouter, + user_api_key_dict: UserAPIKeyAuth | None, +) -> bool: + """Whether the caller may call a canonically-resolved target model group. + + AND-on-target: the caller must be permitted to call the *resolved* group. + The requested spelling having passed the earlier auth check is not enough -- + without this, a key whose allowlist holds only a stale, unserved name would + ride the rewrite onto a deployment it was never granted. (Auth ran on the + requested string before routing, when the target group was not yet known.) + + A denial returns False rather than raising, so the request falls through to + the same 400 an unresolvable model gets today and the response reveals + nothing about the target's existence. + """ + if user_api_key_dict is None: + return True + from litellm.proxy.auth.auth_checks import ( + can_key_call_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed + ) + + try: + await can_key_call_model( + model=canonical_target, + llm_model_list=llm_router.get_model_list(), + valid_token=user_api_key_dict, + llm_router=llm_router, + ) + except Exception: # noqa: BLE001 # any auth failure declines the rewrite; never widens access + return False + return True + + _shared_session_lock: asyncio.Lock | None = None @@ -687,31 +733,24 @@ async def route_request( # routing; the target group was not visible to it then.) On denial # the rewrite is simply declined -- the request falls through to # the same 400 it gets today, revealing nothing about the target. - target_allowed = True - if user_api_key_dict is not None: - from litellm.proxy.auth.auth_checks import ( - can_key_call_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed - ) - - try: - await can_key_call_model( - model=canonical_target, - llm_model_list=llm_router.get_model_list(), - valid_token=user_api_key_dict, - llm_router=llm_router, - ) - except Exception: # noqa: BLE001 # any auth failure declines the rewrite; never widens access - target_allowed = False + target_allowed: Final = await _canonical_target_is_allowed( + canonical_target=canonical_target, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + ) if target_allowed: # Preserve the client's spelling for spend logs / debugging -- # after the rewrite it is otherwise invisible downstream. metadata_field: Final = "litellm_metadata" if "litellm_metadata" in data else "metadata" - existing_metadata: object = data.get(metadata_field) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict + existing_metadata: Final[object] = data.get(metadata_field) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - data is an untyped request dict + # `data` is rewritten in place here because every other route in + # this function does the same; see the rebind-ok notes below. if isinstance(existing_metadata, dict): existing_metadata.setdefault("requested_model", requested_model) # pyright: ignore[reportUnknownMemberType] - metadata dict is untyped else: - data[metadata_field] = {"requested_model": requested_model} - data["model"] = canonical_target + stamp: Final = _requested_model_metadata(requested_model) + data[metadata_field] = stamp # rebind-ok: in-place rewrite, as everywhere in route_request + data["model"] = canonical_target # rebind-ok: in-place data rewrite, as everywhere in route_request return getattr(llm_router, f"{route_type}")(**data) # if no route found then it's a bad request diff --git a/litellm/router.py b/litellm/router.py index aee52e50f7b..3b0e6eb64a4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -636,7 +636,7 @@ class Router: # Lazily-built (provider, canonical_name) -> model_group index for # ``resolve_canonical_model_name``. Invalidated alongside the model # group info cache and on cost-map mutation (generation counter). - self._canonical_model_index: dict[tuple[str, str], str | None] | None = None + self._canonical_model_index: Mapping[tuple[str, str], str | None] | None = None self._canonical_model_index_cost_generation: int = -1 # (requested, target) pairs already announced at INFO, so a hot path # logs once per distinct pair rather than once per request. Keyed on @@ -645,7 +645,8 @@ class Router: # the signal used to size demand for possible follow-up resolution # rules, so a second spelling silently sharing the first's log line # would undercount it. - self._canonical_resolution_logged: set[tuple[str, str]] = set() + # mutable-ok: grow-only log-dedup ledger, intentionally accumulates across requests + self._canonical_resolution_logged: set[tuple[str, str]] = set() # mutable-ok: grow-only log-dedup ledger self._init_routing_groups(None) self.deployment_affinity_ttl_seconds = deployment_affinity_ttl_seconds @@ -10711,7 +10712,7 @@ class Router: """ return resolve_model_group_alias(self.model_group_alias, model) - def _get_canonical_model_index(self) -> dict[tuple[str, str], str | None]: + def _get_canonical_model_index(self) -> Mapping[tuple[str, str], str | None]: """The ``(provider, canonical_name) -> model group`` index, built on demand. Rebuilt when the model list changes (the index is dropped by @@ -10721,10 +10722,10 @@ class Router: cost_generation: Final = get_model_cost_mutation_generation() if self._canonical_model_index is None or self._canonical_model_index_cost_generation != cost_generation: try: - self._canonical_model_index = build_canonical_index(cast("list[DeploymentTypedDict]", self.model_list)) + self._canonical_model_index = build_canonical_index(self.model_list) except Exception as exc: # noqa: BLE001 # index construction must never brick a router; degrade to 'strict' verbose_router_logger.error("canonical-resolution: index build failed, disabling feature: %s", exc) - self._canonical_model_index = {} + self._canonical_model_index = MappingProxyType({}) self._canonical_model_index_cost_generation = cost_generation return self._canonical_model_index diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py index 82344910bc9..71a8a4add94 100644 --- a/litellm/router_utils/canonical_model_resolution.py +++ b/litellm/router_utils/canonical_model_resolution.py @@ -35,12 +35,12 @@ succeeds today can never be re-pointed by this module. See also ``Router.resolve_canonical_model_name``. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import Final import litellm from litellm._logging import verbose_router_logger -from litellm.types.router import DeploymentTypedDict # Cost-map fields that must match exactly for two names to be called the same # model. Pricing equality is a tripwire against false identity, not the @@ -59,10 +59,13 @@ _IDENTITY_ATTESTING_FIELDS: Final[tuple[str, ...]] = ( # billing path the operator never sanctioned. _AMBIGUOUS: Final = None +# Read-only stand-in for a missing sub-mapping on a deployment row. +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) + def _cost_map_entry(model: str) -> Mapping[str, object] | None: """The cost-map entry for ``model``, or None when absent.""" - entry = litellm.model_cost.get(model) + entry: Final = litellm.model_cost.get(model) return entry if isinstance(entry, Mapping) else None @@ -138,8 +141,8 @@ def _undated_variants(model: str) -> tuple[str, ...]: def build_canonical_index( - deployments: list[DeploymentTypedDict], -) -> dict[tuple[str, str], str | None]: + deployments: Sequence[Mapping[str, object]], +) -> Mapping[tuple[str, str], str | None]: """Map ``(provider, canonical_name) -> model group`` for ``deployments``. A model group is indexed only when every one of its deployments agrees on @@ -160,25 +163,24 @@ def build_canonical_index( Never raises: a malformed deployment or cost-map entry degrades to a smaller index, never to a router that fails to boot. """ - index: dict[tuple[str, str], str | None] = {} - group_identity: dict[str, tuple[str, str] | None] = {} + # Both accumulate across the deployment scan, then the result is frozen into + # a MappingProxyType before it leaves this function. + index: Final[dict[tuple[str, str], str | None]] = {} # mutable-ok: local accumulator, frozen on return + group_identity: Final[dict[str, tuple[str, str] | None]] = {} # mutable-ok: local accumulator, never escapes for deployment in deployments: try: - model_info = deployment.get("model_info") or {} - if isinstance( # pyright: ignore[reportUnnecessaryIsInstance] - config/DB rows can violate the TypedDict - model_info, Mapping - ) and model_info.get("team_id"): + model_info = deployment.get("model_info") or _EMPTY # rebind-ok: per-deployment loop variable + if isinstance(model_info, Mapping) and model_info.get("team_id"): continue - # ``model_name``/``model`` are typed Required[str], but this index is - # built from operator config and DB rows that can violate the type, - # so both are validated at runtime rather than trusted. - model_group: object = deployment.get("model_name") - litellm_params = deployment.get("litellm_params") or {} - underlying: object = litellm_params.get("model") - if not isinstance(model_group, str) or not isinstance( # pyright: ignore[reportUnnecessaryIsInstance] - config/DB rows can violate the TypedDict - underlying, str - ): + # Deployments come from operator config and DB rows, so the declared + # str types are validated at runtime rather than trusted. + model_group = deployment.get("model_name") # rebind-ok: per-deployment loop variable + litellm_params = deployment.get("litellm_params") or _EMPTY # rebind-ok: per-deployment loop variable + underlying = ( + litellm_params.get("model") if isinstance(litellm_params, Mapping) else None + ) # rebind-ok: per-deployment loop variable + if not isinstance(model_group, str) or not isinstance(underlying, str): continue identity = canonicalize(underlying) @@ -197,18 +199,24 @@ def build_canonical_index( continue provider, canonical_name = identity # Index the canonical spelling plus any dated<->undated sibling the cost - # map attests is the same model. - spellings: list[str] = [canonical_name] - spellings.extend( + # map attests is the same model: the undated form of a dated canonical + # name, and every dated cost-map entry whose undated form is this one. + undated_siblings = tuple( # rebind-ok: per-group loop variable candidate for candidate in _undated_variants(canonical_name) if _same_model_per_cost_map(canonical_name, candidate) ) - for dated, entry in litellm.model_cost.items(): - if not isinstance(dated, str) or not isinstance(entry, Mapping) or dated in spellings: - continue - if _undated_variants(dated) == (canonical_name,) and _same_model_per_cost_map(canonical_name, dated): - spellings.append(dated) + dated_siblings = tuple( # rebind-ok: per-group loop variable + dated + for dated, entry in litellm.model_cost.items() + if isinstance(dated, str) + and isinstance(entry, Mapping) + and dated != canonical_name + and dated not in undated_siblings + and _undated_variants(dated) == (canonical_name,) + and _same_model_per_cost_map(canonical_name, dated) + ) + spellings = (canonical_name, *undated_siblings, *dated_siblings) # rebind-ok: per-group loop variable for spelling in spellings: key = (provider, spelling) @@ -228,7 +236,7 @@ def build_canonical_index( model_group, ) - return index + return MappingProxyType(index) def lookup( From 9b1fdc1d8d98d0ecd545c0ee5a5a05ef5900390b Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 12:50:02 -0700 Subject: [PATCH 7/8] fix: address three bugbot findings (team re-auth, provider override, proxy settings) All three confirmed real and reproduced before fixing. 1. [High] Team allowlist bypassed AND-on-target. The rewrite re-auth called can_key_call_model, which checks only the key's own allowlist, so team, team-member, and project allowlists were never re-checked against the resolved target. An unrestricted key on a team whose allowlist held only a stale unserved name could ride the rewrite onto a deployment that team was never granted. Now calls can_key_call_resolved_model -- the helper every other post-resolution auth site already uses (model_group_alias rewrites, realtime endpoints, auto-router), which runs the full key/team/member/project chain. 2. [Medium] Index ignored the deployment's custom_llm_provider, canonicalizing only litellm_params.model. Reproduced: a deployment with model='claude-haiku-4-5' + custom_llm_provider='openrouter' was indexed under ('anthropic', ...), so an Anthropic-form request resolved onto OpenRouter credentials, quota, and billing -- precisely the cross-provider hop rule 2 of the module docstring forbids. canonicalize() now takes the override and prefers it over the provider inferred from the model string. 3. [Medium] Operators could not select model_name_resolution: strict on the proxy. Both proxy Router construction sites passed a hardcoded RouterGeneralSettings(async_only_mode=True) as an explicit keyword alongside **router_params. Since router_general_settings IS an accepted config key, setting it raised TypeError: got multiple values for keyword argument at startup -- so the documented opt-out was unreachable either way. Added _proxy_router_general_settings(), which forces async_only_mode (a proxy runtime requirement) while preserving every other operator-set field, and copies rather than mutating the caller's object. Added 10 regression tests. Verified the two re-auth tests actually fail against the pre-fix code (2 failed / 1 passed) rather than passing vacuously. Gates: pytest 531 passed - ruff format clean - ruff check clean - strict BLE001/PERF401 at base parity (2957/22) - type-discipline all LIT rules at base parity - basedpyright new module 0 errors - router_code_coverage 0.0% untested - proxy_server imports cleanly. Co-Authored-By: Claude --- litellm/proxy/proxy_server.py | 42 ++++- litellm/proxy/route_llm_request.py | 15 +- .../canonical_model_resolution.py | 21 ++- .../test_router_canonical_model_resolution.py | 153 ++++++++++++++++++ 4 files changed, 219 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 377963f1b91..22f55406e37 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4057,6 +4057,30 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int: return fetched_model_count +def _proxy_router_general_settings( + configured: RouterGeneralSettings | Mapping[str, Any] | None, +) -> RouterGeneralSettings: + """The proxy's RouterGeneralSettings, preserving operator config. + + ``async_only_mode`` is a proxy-runtime requirement -- only async clients are + initialised on this path -- so it is always forced on. Every other field the + operator set under ``router_settings.router_general_settings`` is kept. + + Before this existed the proxy passed a hardcoded + ``RouterGeneralSettings(async_only_mode=True)`` as an explicit keyword + alongside ``**router_params``, so an operator who set the key in config hit + "got multiple values for keyword argument" at startup and had no way to set + proxy-side router general settings at all (e.g. ``model_name_resolution``). + """ + if configured is None: + return RouterGeneralSettings(async_only_mode=True) + settings: Final = ( + RouterGeneralSettings(**configured) if isinstance(configured, Mapping) else configured.model_copy() + ) + settings.async_only_mode = True + return settings + + class ProxyConfig: """ Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. @@ -5332,13 +5356,18 @@ class ProxyConfig: verbose_proxy_logger.warning( "Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k ) + # `async_only_mode` is a proxy-runtime requirement (only async clients are + # initialised here), so it is forced on. Everything else the operator set + # under `router_settings.router_general_settings` -- e.g. + # `model_name_resolution: strict` -- is preserved; passing the key in + # config used to collide with this keyword and raise TypeError at startup. + router_params["router_general_settings"] = _proxy_router_general_settings( + router_params.get("router_general_settings") + ) router = litellm.Router( **router_params, assistants_config=assistants_config, search_tools=search_tools, - router_general_settings=RouterGeneralSettings( - async_only_mode=True # only init async clients - ), ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid ) @@ -5792,9 +5821,10 @@ class ProxyConfig: verbose_proxy_logger.debug("_model_list: %s", _model_list) llm_router = litellm.Router( model_list=_model_list, - router_general_settings=RouterGeneralSettings( - async_only_mode=True # only init async clients - ), + # DB-sourced model list: no config router_settings in scope + # here, so this is the proxy default (async_only_mode on, + # everything else at its RouterGeneralSettings default). + router_general_settings=_proxy_router_general_settings(None), search_tools=search_tools, ignore_invalid_deployments=True, ) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 507006d4c91..b9784de5bee 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -250,6 +250,15 @@ async def _canonical_target_is_allowed( ride the rewrite onto a deployment it was never granted. (Auth ran on the requested string before routing, when the target group was not yet known.) + Uses ``can_key_call_resolved_model`` -- the same helper every other + post-resolution auth site uses (model_group_alias rewrites, realtime + endpoints, auto-router) -- so the key, team, team-member, and project + allowlists are all re-checked against the target. Checking only the key + (``can_key_call_model``) would leave a team whose allowlist holds a stale + unserved name able to ride the rewrite onto a deployment it was never + granted, since an unrestricted *key* on that team passes the key-level + check on its own. + A denial returns False rather than raising, so the request falls through to the same 400 an unresolvable model gets today and the response reveals nothing about the target's existence. @@ -257,13 +266,13 @@ async def _canonical_target_is_allowed( if user_api_key_dict is None: return True from litellm.proxy.auth.auth_checks import ( - can_key_call_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed + can_key_call_resolved_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed ) try: - await can_key_call_model( + await can_key_call_resolved_model( model=canonical_target, - llm_model_list=llm_router.get_model_list(), + llm_model_list=llm_router.model_list, valid_token=user_api_key_dict, llm_router=llm_router, ) diff --git a/litellm/router_utils/canonical_model_resolution.py b/litellm/router_utils/canonical_model_resolution.py index 71a8a4add94..aa0a8341a6b 100644 --- a/litellm/router_utils/canonical_model_resolution.py +++ b/litellm/router_utils/canonical_model_resolution.py @@ -96,7 +96,7 @@ def _infer_provider(model: str) -> str | None: return custom_llm_provider or None -def canonicalize(model: str) -> tuple[str, str] | None: +def canonicalize(model: str, custom_llm_provider: str | None = None) -> tuple[str, str] | None: """Reduce ``model`` to a ``(provider, canonical_name)`` identity. The canonical name is the model string with any LiteLLM provider-route @@ -104,11 +104,19 @@ def canonicalize(model: str) -> tuple[str, str] | None: LiteLLM's own routing syntax rather than part of the model's identity. The provider is carried alongside so equality checks are always provider-scoped. + ``custom_llm_provider`` -- the deployment's explicit provider override -- + wins over whatever the model string implies. A deployment can carry a + first-party-looking id (``claude-haiku-4-5``) while actually being served + through Bedrock, Vertex, or OpenRouter; inferring the provider from the + string alone would index it as Anthropic and let an Anthropic-form request + be rewritten onto that other provider's credentials, quota, and bill -- + exactly the cross-provider hop rule 2 in the module docstring forbids. + Returns None when the provider cannot be inferred. """ if not model: return None - provider: Final = _infer_provider(model) + provider: Final = custom_llm_provider or _infer_provider(model) if provider is None: return None # get_llm_provider returns the model with its routing prefix stripped, which @@ -183,7 +191,14 @@ def build_canonical_index( if not isinstance(model_group, str) or not isinstance(underlying, str): continue - identity = canonicalize(underlying) + # An explicit provider override decides the provider; see canonicalize(). + provider_override = ( + litellm_params.get("custom_llm_provider") if isinstance(litellm_params, Mapping) else None + ) # rebind-ok: per-deployment loop variable + identity = canonicalize( + underlying, + custom_llm_provider=provider_override if isinstance(provider_override, str) else None, + ) if model_group in group_identity and group_identity[model_group] != identity: # Deployments in this group disagree about what they serve; the # group cannot stand for a single canonical identity. diff --git a/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py index 93109f56d04..8953e949b67 100644 --- a/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py +++ b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py @@ -157,6 +157,47 @@ class TestBuildIndexAndLookup: assert lookup(index, DATED) is None assert lookup(index, UNDATED) is None + def test_custom_llm_provider_override_decides_provider(self): + """Regression (I2): a deployment's explicit custom_llm_provider wins over + whatever the model string implies. A first-party-looking id served via + OpenRouter/Bedrock/Vertex must not be indexed as Anthropic -- otherwise + an Anthropic-form request rides the rewrite onto that other provider's + credentials, quota, and bill, which is exactly the cross-provider hop + rule 2 forbids.""" + index = build_canonical_index( + [ + { + "model_name": "haiku-via-openrouter", + "litellm_params": { + "model": "claude-haiku-4-5", + "custom_llm_provider": "openrouter", + }, + }, + ] + ) + # Indexed under the real provider, not the one the string implies. + assert ("openrouter", UNDATED) in index + assert ("anthropic", UNDATED) not in index + # An Anthropic-form request must not reach the OpenRouter deployment. + assert lookup(index, DATED) is None + assert lookup(index, UNDATED) is None + + def test_custom_llm_provider_override_still_resolves_within_provider(self): + """The override narrows the provider, it does not disable resolution: + a request that infers to the same overridden provider still resolves.""" + index = build_canonical_index( + [ + { + "model_name": "haiku-via-bedrock", + "litellm_params": { + "model": "claude-haiku-4-5", + "custom_llm_provider": "bedrock", + }, + }, + ] + ) + assert index[("bedrock", UNDATED)] == "haiku-via-bedrock" + def test_team_owned_deployment_never_indexed(self): """Regression: a team-owned deployment (model_info.team_id set) must never enter the global canonical index. Without this, a no-team key @@ -410,3 +451,115 @@ class TestAuthAndOnTarget: ) is True ) + + +class TestCanonicalTargetReAuth: + """The rewrite's re-auth must use the *resolved-model* helper. + + Regression: the hook originally called ``can_key_call_model``, which checks + only the key's own allowlist. Team, team-member, and project allowlists were + therefore never re-checked against the resolved target, so an unrestricted + key on a team whose allowlist held only a stale unserved name could ride the + rewrite onto a deployment that team was never granted. + ``can_key_call_resolved_model`` is the helper every other post-resolution + auth site uses (model_group_alias rewrites, realtime endpoints, auto-router). + """ + + @pytest.mark.asyncio + async def test_uses_resolved_model_helper_so_team_scope_is_rechecked(self, anthropic_router: Router): + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.route_llm_request import _canonical_target_is_allowed + + with patch( + "litellm.proxy.auth.auth_checks.can_key_call_resolved_model", + new=AsyncMock(return_value=None), + ) as resolved_check: + allowed = await _canonical_target_is_allowed( + canonical_target=ANTHROPIC_GROUP, + llm_router=anthropic_router, + user_api_key_dict=UserAPIKeyAuth(token="t", team_id="team-A"), + ) + + assert allowed is True + resolved_check.assert_awaited_once() + assert resolved_check.await_args.kwargs["model"] == ANTHROPIC_GROUP + + @pytest.mark.asyncio + async def test_denial_declines_rewrite_rather_than_raising(self, anthropic_router: Router): + """A denial must return False (request falls through to the usual 400), + never propagate an exception that would leak the target's existence.""" + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import UserAPIKeyAuth + + from litellm.proxy.route_llm_request import _canonical_target_is_allowed + + with patch( + "litellm.proxy.auth.auth_checks.can_key_call_resolved_model", + new=AsyncMock(side_effect=ProxyException(message="denied", type="auth_error", param=None, code=401)), + ): + allowed = await _canonical_target_is_allowed( + canonical_target=ANTHROPIC_GROUP, + llm_router=anthropic_router, + user_api_key_dict=UserAPIKeyAuth(token="t", team_id="team-A"), + ) + + assert allowed is False + + @pytest.mark.asyncio + async def test_no_auth_context_is_allowed(self, anthropic_router: Router): + """No key context (non-proxy Router use) leaves the rewrite unguarded by + key auth, matching the surrounding call path.""" + from litellm.proxy.route_llm_request import _canonical_target_is_allowed + + assert ( + await _canonical_target_is_allowed( + canonical_target=ANTHROPIC_GROUP, + llm_router=anthropic_router, + user_api_key_dict=None, + ) + is True + ) + + +class TestProxyRouterGeneralSettings: + """Operators must be able to set model_name_resolution on the proxy. + + Regression: the proxy passed a hardcoded RouterGeneralSettings(async_only_mode=True) + as an explicit keyword alongside **router_params, so setting + router_general_settings in config raised "got multiple values for keyword + argument" at startup -- leaving no way to select 'strict'. + """ + + def test_config_settings_preserved_and_async_only_forced(self): + from litellm.proxy.proxy_server import _proxy_router_general_settings + + settings = _proxy_router_general_settings({"model_name_resolution": "strict"}) + assert settings.model_name_resolution == "strict" + assert settings.async_only_mode is True + + def test_async_only_mode_cannot_be_disabled_by_config(self): + from litellm.proxy.proxy_server import _proxy_router_general_settings + + settings = _proxy_router_general_settings({"async_only_mode": False, "model_name_resolution": "strict"}) + assert settings.async_only_mode is True + assert settings.model_name_resolution == "strict" + + def test_none_yields_proxy_default(self): + from litellm.proxy.proxy_server import _proxy_router_general_settings + + settings = _proxy_router_general_settings(None) + assert settings.async_only_mode is True + assert settings.model_name_resolution == "canonical" + + def test_model_instance_is_not_mutated(self): + from litellm.proxy.proxy_server import _proxy_router_general_settings + from litellm.types.router import RouterGeneralSettings + + original = RouterGeneralSettings(async_only_mode=False, model_name_resolution="strict") + settings = _proxy_router_general_settings(original) + assert settings.async_only_mode is True + # The caller's object must not be rewritten at a distance. + assert original.async_only_mode is False From 1b8daea07c1ed83e80bffec35b0f8a614ddfa283 Mon Sep 17 00:00:00 2001 From: abhi Date: Sat, 15 Aug 2026 18:38:07 -0700 Subject: [PATCH 8/8] fix: fail closed when key context is absent (veria-ai finding) Confirmed real. _canonical_target_is_allowed returned True when user_api_key_dict was None, on the reasoning that no key context meant non-proxy Router use. That reasoning was wrong: of the 12 route_request call sites, only one (common_request_processing, the /chat/completions path) forwards user_api_key_dict. Image generation, rerank, moderation, speech, transcription, realtime, and the Responses WebSocket path are all authenticated endpoints that call route_request without it, so the rewrite ran with no target authorization at all on exactly those paths -- a key whose allowlist held only the stale requested spelling could reach a canonical target it was never granted. Now returns False. Declining costs those endpoints only the convenience rewrite: they behave exactly as they do today, resolution simply never engages, and no request that works now breaks. The AND-on-target guarantee becomes unconditional rather than depending on whether a given caller happens to thread the key through. Verified the primary path is unaffected: with key context present and authorized, resolution still resolves -- that is the Claude Code 403 case this PR exists to fix. Threading user_api_key_dict through the other call sites is the follow-up that re-enables resolution for them; doing it here would touch 11 unrelated endpoints in a PR that should not be changing their signatures. Updated test_no_auth_context_is_allowed -> test_absent_auth_context_fails_closed with the reasoning recorded. Gates: pytest 531 passed - ruff format/check clean - type-discipline all LIT rules at base parity - basedpyright new module 0 errors. Co-Authored-By: Claude --- litellm/proxy/route_llm_request.py | 14 +++++++++++++- .../test_router_canonical_model_resolution.py | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index b9784de5bee..bae8049987c 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -262,9 +262,21 @@ async def _canonical_target_is_allowed( A denial returns False rather than raising, so the request falls through to the same 400 an unresolvable model gets today and the response reveals nothing about the target's existence. + + Absent key context fails CLOSED. Most ``route_request`` callers (image + generation, rerank, moderation, speech, transcription, realtime, Responses + WebSocket) are authenticated but do not currently forward + ``user_api_key_dict``, so treating "no key context" as "allowed" would run + the rewrite with no target authorization at all on exactly those paths. + Declining instead costs those endpoints only the convenience rewrite -- + they behave as they do today, resolution simply never engages -- while + keeping the AND-on-target guarantee unconditional. Threading the key + through those call sites is the follow-up that re-enables resolution for + them; until then this must not be the hole through which the check is + skipped. """ if user_api_key_dict is None: - return True + return False from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, # pyright: ignore[reportUnknownVariableType] - auth_checks is partially typed ) diff --git a/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py index 8953e949b67..7be77dc73d9 100644 --- a/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py +++ b/tests/test_litellm/router_utils/test_router_canonical_model_resolution.py @@ -509,9 +509,17 @@ class TestCanonicalTargetReAuth: assert allowed is False @pytest.mark.asyncio - async def test_no_auth_context_is_allowed(self, anthropic_router: Router): - """No key context (non-proxy Router use) leaves the rewrite unguarded by - key auth, matching the surrounding call path.""" + async def test_absent_auth_context_fails_closed(self, anthropic_router: Router): + """Regression: absent key context must DECLINE the rewrite, not allow it. + + Most route_request callers (image generation, rerank, moderation, + speech, transcription, realtime, Responses WebSocket) are authenticated + but don't currently forward user_api_key_dict. Returning True here + would run the rewrite with no target authorization at all on exactly + those paths -- a key whose allowlist holds only the stale requested + spelling could reach a target it was never granted. Declining costs + those endpoints only the convenience rewrite; the AND-on-target + guarantee stays unconditional.""" from litellm.proxy.route_llm_request import _canonical_target_is_allowed assert ( @@ -520,7 +528,7 @@ class TestCanonicalTargetReAuth: llm_router=anthropic_router, user_api_key_dict=None, ) - is True + is False )