Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_skill_marketplaces

litellm/model_prices_and_context_window_backup.json

@@ -45051,8 +45051,8 @@
         "rules": [
             {
                 "name": "bedrock-claude-ids",
-                "pattern": "anthropic\\.claude-",
-                "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
+                "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-",
+                "description": "A Bedrock-syntax Claude id, for every version: anthropic.claude- at the start of the name, optionally behind a single dotted geo segment (us./eu./au./jp./apac./global./us-gov.). Anchored to the start because routing rules see the raw request string and provider inference feeds the proxy's provider/* wildcard access checks: an id under an unrecognized namespace such as bedrockz/anthropic.claude-... must stay unroutable rather than resolve to bedrock and slip through a bedrock/* key. Routes to bedrock before the bare-id Anthropic rule is consulted.",
                 "model_info": {
                     "litellm_provider": "bedrock"
                 }

litellm/proxy/auth/auth_checks.py

@@ -4383,14 +4383,23 @@ def _model_custom_llm_provider_matches_wildcard_pattern(model: str, allowed_mode
     or
     - `model=claude-3-5-sonnet-20240620`
     - `allowed_model_pattern=anthropic/*`
+
+    A model that already carries a namespace get_llm_provider did not consume
+    (e.g. `bedrockz/anthropic.claude-...`) is never granted here: its provider was
+    inferred from a fragment of the full string, so rebuilding
+    `{provider}/{model}` would produce `bedrock/bedrockz/...` and slip an
+    unrecognized namespace through a `bedrock/*` key.
     """
     try:
-        model, custom_llm_provider, _, _ = get_llm_provider(model=model)
+        stripped_model, custom_llm_provider, _, _ = get_llm_provider(model=model)
     except Exception:
         return False
 
+    if stripped_model == model and "/" in model:
+        return False
+
     return is_model_allowed_by_pattern(
-        model=f"{custom_llm_provider}/{model}",
+        model=f"{custom_llm_provider}/{stripped_model}",
         allowed_model_pattern=allowed_model_pattern,
     )
 

litellm/router_strategy/complexity_router/complexity_router.py

@@ -14,6 +14,7 @@ Inspired by ClawRouter: https://github.com/BlockRunAI/ClawRouter
 """
 
 import asyncio
+import random
 import re
 from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
 
@@ -437,22 +438,26 @@ class ComplexityRouter(CustomLogger):
         """
         tier_key = tier.value if isinstance(tier, ComplexityTier) else tier
 
-        # Check config tiers mapping
-        model = self.config.tiers.get(tier_key)
-        if model:
-            return model
+        if tier_key in self.config.tiers:
+            return self._pick_from_tier_value(self.config.tiers[tier_key], tier_key)
 
-        # Fallback to default model if configured
         if self.config.default_model:
             return self.config.default_model
 
-        # Last resort: return MEDIUM tier model or error
-        medium_model = self.config.tiers.get(ComplexityTier.MEDIUM.value)
-        if medium_model:
-            return medium_model
+        medium_key = ComplexityTier.MEDIUM.value
+        if medium_key in self.config.tiers:
+            return self._pick_from_tier_value(self.config.tiers[medium_key], medium_key)
 
         raise ValueError(f"No model configured for tier {tier_key} and no default_model set")
 
+    @staticmethod
+    def _pick_from_tier_value(model: str | list[str], tier_key: str) -> str:
+        if isinstance(model, str):
+            return model
+        if not model:
+            raise ValueError(f"Empty model pool for tier {tier_key}")
+        return random.choice(model)
+
     def _lexical_tier_override(self, user_message: str) -> Optional[ComplexityTier]:
         """When keyword_tier_rules match literally, the most-severe matched tier wins.
 

litellm/router_strategy/complexity_router/config.py

@@ -8,7 +8,7 @@ All values are configurable via proxy config.yaml.
 from enum import Enum
 from typing import Dict, List, Literal, Optional
 
-from pydantic import BaseModel, ConfigDict, Field, model_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 
 class ComplexityTier(str, Enum):
@@ -244,10 +244,12 @@ class ClassifierLLMConfig(BaseModel):
 class ComplexityRouterConfig(BaseModel):
     """Configuration for the ComplexityRouter."""
 
-    # Tier to model mapping
-    tiers: Dict[str, str] = Field(
+    # string = pin; list = random pick from the tier pool
+    tiers: dict[str, str | list[str]] = Field(
         default_factory=lambda: DEFAULT_TIER_MODELS.copy(),
-        description="Mapping of complexity tiers to model names",
+        description=(
+            "Mapping of complexity tiers to a model or model pool. A list is randomly picked from for that tier"
+        ),
     )
 
     # Tier boundaries (normalized scores)
@@ -335,6 +337,21 @@ class ComplexityRouterConfig(BaseModel):
 
     model_config = ConfigDict(extra="allow")  # Allow additional fields
 
+    @field_validator("tiers", mode="before")
+    @classmethod
+    def _coerce_tier_values(cls, value: object) -> object:
+        if not isinstance(value, dict):
+            return value
+        coerced: dict[str, object] = {}
+        for key, item in value.items():
+            if isinstance(item, str):
+                coerced[key] = item
+            elif isinstance(item, (list, tuple)):
+                coerced[key] = list(item)
+            else:
+                coerced[key] = item
+        return coerced
+
     @model_validator(mode="after")
     def _validate_llm_classifier_config(self) -> "ComplexityRouterConfig":
         if self.classifier_type == "llm" and self.classifier_llm_config is None:

model_prices_and_context_window.json

@@ -45284,8 +45284,8 @@
         "rules": [
             {
                 "name": "bedrock-claude-ids",
-                "pattern": "anthropic\\.claude-",
-                "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
+                "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-",
+                "description": "A Bedrock-syntax Claude id, for every version: anthropic.claude- at the start of the name, optionally behind a single dotted geo segment (us./eu./au./jp./apac./global./us-gov.). Anchored to the start because routing rules see the raw request string and provider inference feeds the proxy's provider/* wildcard access checks: an id under an unrecognized namespace such as bedrockz/anthropic.claude-... must stay unroutable rather than resolve to bedrock and slip through a bedrock/* key. Routes to bedrock before the bare-id Anthropic rule is consulted.",
                 "model_info": {
                     "litellm_provider": "bedrock"
                 }

tests/proxy_unit_tests/test_auth_checks.py

@@ -236,6 +236,8 @@ async def test_can_team_call_model(model, expect_to_work):
         (["bedrock/*"], "bedrock/anthropic.claude-3-5-sonnet-20240620", True),
         (["bedrock/*"], "bedrockz/anthropic.claude-3-5-sonnet-20240620", False),
         (["bedrock/us.*"], "bedrock/us.amazon.nova-micro-v1:0", True),
+        (["openai/*"], "ft:gpt-4-0613", True),
+        (["openai/*"], "bedrockz/ft:gpt-4-0613", False),
     ],
 )
 @pytest.mark.asyncio

tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py

@@ -138,6 +138,42 @@ def test_shipped_backup_carries_the_claude_routing_rules():
         set_fallback_generalizations(previous)
 
 
+def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
+    """Routing rules decide ``litellm_provider`` for otherwise-unknown ids, and the
+    proxy's wildcard access check (``can_key_call_model`` with a ``bedrock/*`` key)
+    trusts that inference: it rebuilds ``{provider}/{model}`` and matches it against
+    the key's patterns. A routing pattern that matches as a substring lets
+    ``bedrockz/anthropic.claude-...`` resolve to bedrock and slip through a
+    ``bedrock/*`` key, so every shipped routing rule must anchor to the start of
+    the name and never match an id carrying an unrecognized namespace prefix."""
+    backup = GetModelCostMap.load_local_model_cost_map()
+    rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
+
+    routing_rules = [r for r in rules if "litellm_provider" in r["model_info"]]
+    assert routing_rules
+    assert all(r["pattern"].startswith("^") for r in routing_rules)
+
+    previous = list(get_fallback_generalization_rules())
+    try:
+        set_fallback_generalizations(rules)
+        for bedrock_id in [
+            "anthropic.claude-3-5-sonnet-20240620-v1:0",
+            "anthropic.claude-v2:1",
+            "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+            "us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0",
+            "global.anthropic.claude-fable-5-20260120-v1:0",
+        ]:
+            assert match_routing_generalization(bedrock_id) == "bedrock", bedrock_id
+        for namespaced in [
+            "bedrockz/anthropic.claude-3-5-sonnet-20240620",
+            "bedrockz/us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+            "bedrockz/claude-3-5-sonnet-20240620",
+        ]:
+            assert match_routing_generalization(namespaced) is None, namespaced
+    finally:
+        set_fallback_generalizations(previous)
+
+
 def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
     """Adaptive thinking is data, not code. The bundled backup must carry
     supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider

tests/test_litellm/router_strategy/test_complexity_router.py

@@ -315,6 +315,36 @@ class TestModelSelection:
         model = router.get_model_for_tier(ComplexityTier.SIMPLE)
         assert model == "fallback-model"
 
+    def test_get_model_for_tier_list_random_choice(self, mock_router_instance):
+        router = ComplexityRouter(
+            model_name="test-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config={
+                "tiers": {"SIMPLE": ["cheap", "premium"], "MEDIUM": "mid"},
+                "default_model": "mid",
+            },
+        )
+        pool = ["cheap", "premium"]
+        with patch(
+            "litellm.router_strategy.complexity_router.complexity_router.random.choice",
+            return_value="premium",
+        ) as choice:
+            assert router.get_model_for_tier(ComplexityTier.SIMPLE) == "premium"
+            choice.assert_called_once_with(pool)
+        assert router.get_model_for_tier(ComplexityTier.MEDIUM) == "mid"
+
+    def test_get_model_for_tier_empty_pool_raises(self, mock_router_instance):
+        router = ComplexityRouter(
+            model_name="test-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config={
+                "tiers": {"SIMPLE": []},
+                "default_model": "mid",
+            },
+        )
+        with pytest.raises(ValueError, match="Empty model pool for tier SIMPLE"):
+            router.get_model_for_tier(ComplexityTier.SIMPLE)
+
 
 class TestPreRoutingHook:
     """Test the async_pre_routing_hook method."""