From 2ed4ceb12e5ec5867d150a0d0eb5b9a97196cd4a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:00:03 -0700 Subject: [PATCH 1/3] fix(model-cost-map): anchor the bedrock-claude-ids routing rule to the start of the id --- ...odel_prices_and_context_window_backup.json | 4 +-- model_prices_and_context_window.json | 4 +-- .../test_get_model_cost_map.py | 36 +++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b4ae842c4e6..773ffb92b59 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45051,8 +45051,8 @@ "rules": [ { "name": "bedrock-claude-ids", - "pattern": "anthropic\\.claude-", - "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.", + "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-", + "description": "A Bedrock-syntax Claude id, for every version: anthropic.claude- at the start of the name, optionally behind a single dotted geo segment (us./eu./au./jp./apac./global./us-gov.). Anchored to the start because routing rules see the raw request string and provider inference feeds the proxy's provider/* wildcard access checks: an id under an unrecognized namespace such as bedrockz/anthropic.claude-... must stay unroutable rather than resolve to bedrock and slip through a bedrock/* key. Routes to bedrock before the bare-id Anthropic rule is consulted.", "model_info": { "litellm_provider": "bedrock" } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 94d9f6496bd..6a770998331 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45284,8 +45284,8 @@ "rules": [ { "name": "bedrock-claude-ids", - "pattern": "anthropic\\.claude-", - "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.", + "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-", + "description": "A Bedrock-syntax Claude id, for every version: anthropic.claude- at the start of the name, optionally behind a single dotted geo segment (us./eu./au./jp./apac./global./us-gov.). Anchored to the start because routing rules see the raw request string and provider inference feeds the proxy's provider/* wildcard access checks: an id under an unrecognized namespace such as bedrockz/anthropic.claude-... must stay unroutable rather than resolve to bedrock and slip through a bedrock/* key. Routes to bedrock before the bare-id Anthropic rule is consulted.", "model_info": { "litellm_provider": "bedrock" } diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index bdd71f28b1a..1a38b5dc769 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -138,6 +138,42 @@ def test_shipped_backup_carries_the_claude_routing_rules(): set_fallback_generalizations(previous) +def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace(): + """Routing rules decide ``litellm_provider`` for otherwise-unknown ids, and the + proxy's wildcard access check (``can_key_call_model`` with a ``bedrock/*`` key) + trusts that inference: it rebuilds ``{provider}/{model}`` and matches it against + the key's patterns. A routing pattern that matches as a substring lets + ``bedrockz/anthropic.claude-...`` resolve to bedrock and slip through a + ``bedrock/*`` key, so every shipped routing rule must anchor to the start of + the name and never match an id carrying an unrecognized namespace prefix.""" + backup = GetModelCostMap.load_local_model_cost_map() + rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] + + routing_rules = [r for r in rules if "litellm_provider" in r["model_info"]] + assert routing_rules + assert all(r["pattern"].startswith("^") for r in routing_rules) + + previous = list(get_fallback_generalization_rules()) + try: + set_fallback_generalizations(rules) + for bedrock_id in [ + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "anthropic.claude-v2:1", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0", + "global.anthropic.claude-fable-5-20260120-v1:0", + ]: + assert match_routing_generalization(bedrock_id) == "bedrock", bedrock_id + for namespaced in [ + "bedrockz/anthropic.claude-3-5-sonnet-20240620", + "bedrockz/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrockz/claude-3-5-sonnet-20240620", + ]: + assert match_routing_generalization(namespaced) is None, namespaced + finally: + set_fallback_generalizations(previous) + + def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): """Adaptive thinking is data, not code. The bundled backup must carry supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider From d0d1c0e346fdb5f907666b1fb4aa0b26f21beb9c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:00:03 -0700 Subject: [PATCH 2/3] fix(proxy-auth): deny provider-wildcard access inferred through an unrecognized model namespace --- litellm/proxy/auth/auth_checks.py | 13 +++++++++++-- tests/proxy_unit_tests/test_auth_checks.py | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 230b9b70ff0..93811812901 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4383,14 +4383,23 @@ def _model_custom_llm_provider_matches_wildcard_pattern(model: str, allowed_mode or - `model=claude-3-5-sonnet-20240620` - `allowed_model_pattern=anthropic/*` + + A model that already carries a namespace get_llm_provider did not consume + (e.g. `bedrockz/anthropic.claude-...`) is never granted here: its provider was + inferred from a fragment of the full string, so rebuilding + `{provider}/{model}` would produce `bedrock/bedrockz/...` and slip an + unrecognized namespace through a `bedrock/*` key. """ try: - model, custom_llm_provider, _, _ = get_llm_provider(model=model) + stripped_model, custom_llm_provider, _, _ = get_llm_provider(model=model) except Exception: return False + if stripped_model == model and "/" in model: + return False + return is_model_allowed_by_pattern( - model=f"{custom_llm_provider}/{model}", + model=f"{custom_llm_provider}/{stripped_model}", allowed_model_pattern=allowed_model_pattern, ) diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index e7136ecb195..e58e6c9694b 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -236,6 +236,8 @@ async def test_can_team_call_model(model, expect_to_work): (["bedrock/*"], "bedrock/anthropic.claude-3-5-sonnet-20240620", True), (["bedrock/*"], "bedrockz/anthropic.claude-3-5-sonnet-20240620", False), (["bedrock/us.*"], "bedrock/us.amazon.nova-micro-v1:0", True), + (["openai/*"], "ft:gpt-4-0613", True), + (["openai/*"], "bedrockz/ft:gpt-4-0613", False), ], ) @pytest.mark.asyncio From f717e3b2f0d4914ee5311f058a2514d676306988 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Jul 2026 19:27:11 -0700 Subject: [PATCH 3/3] feat(router): random-pick multi-model complexity tiers (#32967) * feat(router): random-pick multi-model complexity tiers Tier pools already make sense without adaptive; stop pinning lists to index 0 and shuffle within the classified tier instead. Co-authored-by: Cursor * fix(ci): format complexity router config Co-authored-by: Cursor * fix(ci): use PEP 585 types for tier pools Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .../complexity_router/complexity_router.py | 23 ++++++++------ .../complexity_router/config.py | 25 +++++++++++++--- .../router_strategy/test_complexity_router.py | 30 +++++++++++++++++++ 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 11719b8a18f..2138a0112a0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -14,6 +14,7 @@ Inspired by ClawRouter: https://github.com/BlockRunAI/ClawRouter """ import asyncio +import random import re from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast @@ -437,22 +438,26 @@ class ComplexityRouter(CustomLogger): """ tier_key = tier.value if isinstance(tier, ComplexityTier) else tier - # Check config tiers mapping - model = self.config.tiers.get(tier_key) - if model: - return model + if tier_key in self.config.tiers: + return self._pick_from_tier_value(self.config.tiers[tier_key], tier_key) - # Fallback to default model if configured if self.config.default_model: return self.config.default_model - # Last resort: return MEDIUM tier model or error - medium_model = self.config.tiers.get(ComplexityTier.MEDIUM.value) - if medium_model: - return medium_model + medium_key = ComplexityTier.MEDIUM.value + if medium_key in self.config.tiers: + return self._pick_from_tier_value(self.config.tiers[medium_key], medium_key) raise ValueError(f"No model configured for tier {tier_key} and no default_model set") + @staticmethod + def _pick_from_tier_value(model: str | list[str], tier_key: str) -> str: + if isinstance(model, str): + return model + if not model: + raise ValueError(f"Empty model pool for tier {tier_key}") + return random.choice(model) + def _lexical_tier_override(self, user_message: str) -> Optional[ComplexityTier]: """When keyword_tier_rules match literally, the most-severe matched tier wins. diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 125de6f7489..8c8e5acb51f 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -8,7 +8,7 @@ All values are configurable via proxy config.yaml. from enum import Enum from typing import Dict, List, Literal, Optional -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class ComplexityTier(str, Enum): @@ -244,10 +244,12 @@ class ClassifierLLMConfig(BaseModel): class ComplexityRouterConfig(BaseModel): """Configuration for the ComplexityRouter.""" - # Tier to model mapping - tiers: Dict[str, str] = Field( + # string = pin; list = random pick from the tier pool + tiers: dict[str, str | list[str]] = Field( default_factory=lambda: DEFAULT_TIER_MODELS.copy(), - description="Mapping of complexity tiers to model names", + description=( + "Mapping of complexity tiers to a model or model pool. A list is randomly picked from for that tier" + ), ) # Tier boundaries (normalized scores) @@ -335,6 +337,21 @@ class ComplexityRouterConfig(BaseModel): model_config = ConfigDict(extra="allow") # Allow additional fields + @field_validator("tiers", mode="before") + @classmethod + def _coerce_tier_values(cls, value: object) -> object: + if not isinstance(value, dict): + return value + coerced: dict[str, object] = {} + for key, item in value.items(): + if isinstance(item, str): + coerced[key] = item + elif isinstance(item, (list, tuple)): + coerced[key] = list(item) + else: + coerced[key] = item + return coerced + @model_validator(mode="after") def _validate_llm_classifier_config(self) -> "ComplexityRouterConfig": if self.classifier_type == "llm" and self.classifier_llm_config is None: diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index f47c19b2baa..e1133620a57 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -315,6 +315,36 @@ class TestModelSelection: model = router.get_model_for_tier(ComplexityTier.SIMPLE) assert model == "fallback-model" + def test_get_model_for_tier_list_random_choice(self, mock_router_instance): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": ["cheap", "premium"], "MEDIUM": "mid"}, + "default_model": "mid", + }, + ) + pool = ["cheap", "premium"] + with patch( + "litellm.router_strategy.complexity_router.complexity_router.random.choice", + return_value="premium", + ) as choice: + assert router.get_model_for_tier(ComplexityTier.SIMPLE) == "premium" + choice.assert_called_once_with(pool) + assert router.get_model_for_tier(ComplexityTier.MEDIUM) == "mid" + + def test_get_model_for_tier_empty_pool_raises(self, mock_router_instance): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": []}, + "default_model": "mid", + }, + ) + with pytest.raises(ValueError, match="Empty model pool for tier SIMPLE"): + router.get_model_for_tier(ComplexityTier.SIMPLE) + class TestPreRoutingHook: """Test the async_pre_routing_hook method."""