Merge 9b0ffd564c into e4f59a953c

litellm/proxy/proxy_server.py

@@ -9267,9 +9267,18 @@ def select_data_generator(
     )
 
 
-def get_litellm_model_info(model: dict = {}):
+def get_litellm_model_info(model: dict = {}, llm_router: Router | None = None):
     model_info: Final = model.get("model_info", {})
     model_to_lookup = model.get("litellm_params", {}).get("model", None)
+    if _is_auto_router_model(model):
+        window: Final = (
+            llm_router.get_auto_router_context_window(str(model.get("model_name") or ""))
+            if llm_router is not None
+            else None
+        )
+        if window is None:
+            return _EMPTY_MAPPING
+        return MappingProxyType({"max_input_tokens": window})
     try:
         if "azure" in model_to_lookup or model_info.get("base_model"):
             model_to_lookup = model_info.get("base_model", None)
@@ -13536,10 +13545,10 @@ def _enrich_model_info_with_litellm_data(
 
     # read litellm model_prices_and_context_window.json to get the following:
     # input_cost_per_token, output_cost_per_token, max_tokens
-    litellm_model_info = get_litellm_model_info(model=model)
+    litellm_model_info = get_litellm_model_info(model=model, llm_router=llm_router)
 
     # 2nd pass on the model, try seeing if we can find model in litellm model_cost map
-    if litellm_model_info == {}:
+    if litellm_model_info == {} and not _is_auto_router_model(model):
         # use litellm_param model_name to get model_info
         litellm_params = model.get("litellm_params", {})
         litellm_model = litellm_params.get("model", None)
@@ -13548,7 +13557,7 @@ def _enrich_model_info_with_litellm_data(
         except Exception:
             litellm_model_info = {}
     # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map
-    if litellm_model_info == {}:
+    if litellm_model_info == {} and not _is_auto_router_model(model):
         # use litellm_param model_name to get model_info
         litellm_params = model.get("litellm_params", {})
         litellm_model = litellm_params.get("model", None)
@@ -14997,16 +15006,16 @@ def _translate_model_name_for_response(model: dict) -> dict:
     return {**model, "model_name": team_public}
 
 
-def _get_proxy_model_info(model: dict) -> dict:
+def _get_proxy_model_info(model: dict, llm_router: Router | None = None) -> dict:
     # provided model_info in config.yaml
     model_info: Final = model.get("model_info", {})
 
     # read litellm model_prices_and_context_window.json to get the following:
     # input_cost_per_token, output_cost_per_token, max_tokens
-    litellm_model_info = get_litellm_model_info(model=model)
+    litellm_model_info = get_litellm_model_info(model=model, llm_router=llm_router)
 
     # 2nd pass on the model, try seeing if we can find model in litellm model_cost map
-    if litellm_model_info == {}:
+    if litellm_model_info == {} and not _is_auto_router_model(model):
         # use litellm_param model_name to get model_info
         litellm_params = model.get("litellm_params", {})
         litellm_model = litellm_params.get("model", None)
@@ -15015,7 +15024,7 @@ def _get_proxy_model_info(model: dict) -> dict:
         except Exception:
             litellm_model_info = {}
     # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map
-    if litellm_model_info == {}:
+    if litellm_model_info == {} and not _is_auto_router_model(model):
         # use litellm_param model_name to get model_info
         litellm_params = model.get("litellm_params", {})
         litellm_model = litellm_params.get("model", None)
@@ -15163,7 +15172,9 @@ async def model_info_v1(
                 status_code=400,
                 detail={"error": f"Model id = {litellm_model_id} not found on litellm proxy"},
             )
-        _deployment_info_dict = _get_proxy_model_info(model=deployment_info.model_dump(exclude_none=True))
+        _deployment_info_dict = _get_proxy_model_info(
+            model=deployment_info.model_dump(exclude_none=True), llm_router=llm_router
+        )
         single_model_list: list[dict] = [_deployment_info_dict]
         if prisma_client is not None:
             single_model_list = await _populate_team_access_on_models(

litellm/router.py

@@ -10266,15 +10266,39 @@ class Router:
                     model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
                     for model_info, litellm_params in zip(model_infos, params)
                 )
-                if isinstance(key, str) and key
+                if isinstance(key, str) and key and not key.startswith(AUTO_ROUTER_MODEL_PREFIX)
             )
         )
+        configured_input: Final = self._widest_configured_limit(model_infos, "max_input_tokens")
         return DeploymentModelListingInfo(
             cost_map_keys=cost_map_keys,
-            max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),
+            max_input_tokens=(
+                configured_input if configured_input is not None else self.get_auto_router_context_window(model_name)
+            ),
             max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
         )
 
+    def get_auto_router_context_window(self, model_name: str) -> int | None:
+        """The context window an auto-router under ``model_name`` can serve, or None.
+
+        An auto-router deployment is a marker whose ``litellm_params.model`` is absent from the cost
+        map, so every other source a listing consults answers nothing and the router advertises no
+        window at all. The window it can actually serve lives in the tier model groups it dispatches
+        to, and only the strategy knows those.
+
+        Smallest across the strategies registered under the name: a listing carries no request
+        tags, tags select one strategy rather than escalating between them, and a caller whose tags
+        land on the narrowest gets no second chance at a wider one.
+
+        Derived for complexity routers only; the adaptive, quality and semantic strategies hold
+        their candidates in a different shape and are not covered.
+        """
+        tagged: Final = self.complexity_routers.get(model_name) or ()
+        windows: Final = tuple(
+            window for window in (entry.strategy.advertised_context_window() for entry in tagged) if window is not None
+        )
+        return min(windows) if windows else None
+
     @staticmethod
     def _widest_configured_limit(model_infos: Sequence[Mapping[str, Any]], field: str) -> int | None:
         """The largest usable value of ``field`` across a group's configured model_info blocks."""

litellm/router_strategy/complexity_router/complexity_router.py

@@ -2615,6 +2615,29 @@ class ComplexityRouter(CustomLogger):
         deployments: Final = list_models(model_name=group) if callable(list_models) else None
         return tuple(deployments) if isinstance(deployments, list) else ()
 
+    def advertised_context_window(self) -> int | None:
+        """The context window this router can serve, or None when any tier group's window is
+        unresolvable.
+
+        A marker deployment carries no provider metadata, so a listing has nothing to report for it
+        unless the tiers are walked. The direction of each aggregate tracks whether a fit check
+        exists at that level. Within a group it is the smallest, because the core router picks a
+        deployment inside a group without one. Across the tier groups it is the widest only while
+        context-window escalation is on, since that is what moves a prompt off a tier too small to
+        hold it before dispatch; with escalation off the classifier's tier is final, so the only
+        window every request is sure of is the smallest. A group whose window is unknown is never
+        escalated onto and its presence makes the whole answer a guess: report nothing instead.
+        """
+        groups: Final = frozenset(group for pool in self._tier_pools().values() for group in pool)
+        windows: Final = tuple(
+            window
+            for window, unknown in (self._group_window_facts(group) for group in groups)
+            if window is not None and not unknown
+        )
+        if not groups or len(windows) != len(groups):
+            return None
+        return max(windows) if self.config.enable_context_window_escalation else min(windows)
+
     def _group_output_ceiling(self, group: str) -> int | None:
         """Smallest max_output_tokens across the group's deployments, or None when any deployment
         declares none: the core router picks within the group without a fit check, and a ceiling

tests/test_litellm/proxy/proxy_server/test_lifecycle.py

@@ -1073,3 +1073,40 @@ async def test_prometheus_fallback_stats_job_runs_when_the_lock_is_free_or_absen
     await jobs["prometheus_fallback_stats_job"]()
 
     assert send_fallback_stats.await_count == 2
+
+
+def _window_router(tiers: dict[str, str]):
+    import litellm
+
+    return litellm.Router(
+        model_list=[
+            {"model_name": "narrow", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "k"}},
+            {"model_name": "wide", "litellm_params": {"model": "openai/gpt-4.1", "api_key": "k"}},
+            {
+                "model_name": "router",
+                "litellm_params": {
+                    "model": "auto_router/complexity_router",
+                    "complexity_router_config": {"tiers": tiers},
+                    "complexity_router_default_model": "narrow",
+                },
+            },
+        ]
+    )
+
+
+def test_get_litellm_model_info_derives_window_for_an_auto_router_marker():
+    router = _window_router({"SIMPLE": "narrow", "COMPLEX": "wide"})
+    marker = {"model_name": "router", "litellm_params": {"model": "auto_router/complexity_router"}}
+
+    assert get_litellm_model_info(model=marker).get("max_input_tokens") is None
+    assert get_litellm_model_info(model=marker, llm_router=router) == {"max_input_tokens": 1_047_576}
+
+
+def test_get_litellm_model_info_still_prices_a_concrete_deployment_sharing_the_router_name():
+    router = _window_router({"SIMPLE": "narrow", "COMPLEX": "wide"})
+    concrete = {"model_name": "router", "litellm_params": {"model": "openai/gpt-4o-mini"}}
+
+    result = get_litellm_model_info(model=concrete, llm_router=router)
+
+    assert result.get("max_input_tokens") == 128_000
+    assert result.get("input_cost_per_token") is not None

tests/test_litellm/proxy/proxy_server/test_routes_model_info.py

@@ -92,7 +92,7 @@ def configured_router(monkeypatch):
     monkeypatch.setattr(proxy_server, "llm_router", router)
     monkeypatch.setattr(proxy_server, "llm_model_list", [{"model_name": "gpt-4"}])
     monkeypatch.setattr(proxy_server, "user_model", None)
-    monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model: model)
+    monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model, llm_router=None: model)
     yield router
 
 

tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py

@@ -804,7 +804,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce
     monkeypatch.setattr(ps, "llm_model_list", [team_row])
     monkeypatch.setattr(ps, "llm_router", router)
     monkeypatch.setattr(ps, "prisma_client", MagicMock())
-    monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row)
+    monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, llm_router=None: team_row)
     monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate)
 
     caller = UserAPIKeyAuth(
@@ -839,7 +839,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey
     monkeypatch.setattr(ps, "llm_model_list", [team_row])
     monkeypatch.setattr(ps, "llm_router", router)
     monkeypatch.setattr(ps, "prisma_client", MagicMock())
-    monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row)
+    monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, llm_router=None: team_row)
     monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate)
     monkeypatch.setattr(ps, "_filter_models_by_team_id", team_filter)
 

tests/test_litellm/proxy/test_proxy_utils.py

@@ -2328,3 +2328,118 @@ class TestPrismaClientTokenAuthBehindThePool:
         assert isinstance(client.db, RoutingPrismaWrapper)
         assert client.db.writer.iam_token_db_auth is True
         assert client.db.reader.iam_token_db_auth is True
+
+
+def _auto_router_model_list(
+    tiers: dict[str, str | list[str]],
+    router_model_info: dict[str, object] | None = None,
+) -> list[dict]:
+    return [
+        {
+            "model_name": "narrow",
+            "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"},
+        },
+        {
+            "model_name": "wide",
+            "litellm_params": {"model": "openai/gpt-4.1", "api_key": "test-key"},
+        },
+        {
+            "model_name": "unmapped",
+            "litellm_params": {"model": "openai/no-such-model-xyz", "api_key": "test-key"},
+        },
+        {
+            "model_name": "router",
+            "litellm_params": {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": {"tiers": tiers},
+                "complexity_router_default_model": "narrow",
+            },
+            **({"model_info": router_model_info} if router_model_info is not None else {}),
+        },
+    ]
+
+
+def test_create_model_info_response_derives_window_from_auto_router_tiers():
+    router = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow", "COMPLEX": "wide"}))
+
+    response = create_model_info_response(
+        model_id="router",
+        provider="openai",
+        llm_router=router,
+        get_model_info=_raise_unmapped,
+    )
+
+    assert response["max_input_tokens"] == 1_047_576
+
+
+def test_create_model_info_response_auto_router_window_is_widest_across_tiers():
+    """The narrow tier must not decide the advertised window: escalation moves an oversized
+    prompt onto a tier that fits, so the wide end is what the router serves."""
+    narrow_only = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow"}))
+    both = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow", "COMPLEX": "wide"}))
+
+    def window(router: litellm.Router) -> int | None:
+        return create_model_info_response(
+            model_id="router", provider="openai", llm_router=router, get_model_info=_raise_unmapped
+        ).get("max_input_tokens")
+
+    assert window(narrow_only) == 128_000
+    assert window(both) == 1_047_576
+
+
+def test_create_model_info_response_omits_auto_router_window_when_a_tier_is_unresolvable():
+    router = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "wide", "COMPLEX": "unmapped"}))
+
+    response = create_model_info_response(
+        model_id="router",
+        provider="openai",
+        llm_router=router,
+        get_model_info=_raise_unmapped,
+    )
+
+    assert "max_input_tokens" not in response
+
+
+def test_create_model_info_response_auto_router_configured_window_outranks_derivation():
+    router = litellm.Router(
+        model_list=_auto_router_model_list(
+            {"SIMPLE": "narrow", "COMPLEX": "wide"}, router_model_info={"max_input_tokens": 555_000}
+        )
+    )
+
+    response = create_model_info_response(
+        model_id="router",
+        provider="openai",
+        llm_router=router,
+        get_model_info=_raise_unmapped,
+    )
+
+    assert response["max_input_tokens"] == 555_000
+
+
+def test_create_model_info_response_auto_router_window_does_not_leak_between_routers():
+    """Every auto-router marker shares one cost-map key, so a window declared on one router used to
+    come back as the advertised window of every other router in the process."""
+    router = litellm.Router(
+        model_list=[
+            *_auto_router_model_list({"SIMPLE": "wide", "COMPLEX": "unmapped"}),
+            {
+                "model_name": "declared-router",
+                "litellm_params": {
+                    "model": "auto_router/complexity_router",
+                    "complexity_router_config": {"tiers": {"SIMPLE": "narrow"}},
+                    "complexity_router_default_model": "narrow",
+                },
+                "model_info": {"max_input_tokens": 555_000},
+            },
+        ]
+    )
+
+    def window(model_id: str) -> int | None:
+        return create_model_info_response(
+            model_id=model_id, provider="openai", llm_router=router, get_model_info=litellm.get_model_info
+        ).get("max_input_tokens")
+
+    assert litellm.model_cost.get("auto_router/complexity_router", {}).get("max_input_tokens") == 555_000
+    assert window("declared-router") == 555_000
+    assert window("router") is None

tests/test_litellm/router_strategy/test_complexity_router.py

@@ -15184,3 +15184,67 @@ class TestNonReasoningTier:
             "complex",
             "reasoning",
         )
+
+
+class TestAdvertisedContextWindow:
+    """A marker deployment carries no provider metadata, so a listing reports no window for the
+    router unless the tier groups are walked. The direction of each aggregate tracks whether a fit
+    check exists at that level: escalation checks fit between tiers, nothing checks it within a
+    group.
+    """
+
+    @staticmethod
+    def _router(*deployments: tuple, **overrides) -> ComplexityRouter:
+        return ComplexityRouter(
+            model_name="test-router",
+            litellm_router_instance=_windowed_router(*deployments),
+            complexity_router_config=_tier_config(**overrides),
+        )
+
+    def test_reports_the_widest_tier_a_prompt_can_be_escalated_onto(self):
+        router = self._router(_SMALL, _BIG)
+        assert router.advertised_context_window() == 200000
+
+    def test_escalation_off_reports_the_tier_a_request_cannot_be_moved_off(self):
+        """With escalation disabled the classifier's tier is final, so the widest would promise a
+        window that a request classified onto the narrow tier can never reach."""
+        router = self._router(_SMALL, _BIG, enable_context_window_escalation=False)
+        assert router.advertised_context_window() == 16385
+
+    def test_a_single_tier_router_reports_that_tier(self):
+        router = self._router(_SMALL, _BIG, tiers={"SIMPLE": "small-model"})
+        assert router.advertised_context_window() == 16385
+
+    def test_a_group_is_judged_by_its_smallest_member(self):
+        """Two deployments serve one tier and the core router picks between them with no fit
+        check, so the group can only promise the smaller window."""
+        router = self._router(
+            ("mixed", "openai/gpt-4o-mini", 200000),
+            ("mixed", "openai/gpt-3.5-turbo", 16385),
+            tiers={"SIMPLE": "mixed"},
+        )
+        assert router.advertised_context_window() == 16385
+
+    def test_an_unresolvable_tier_makes_the_whole_answer_a_guess(self):
+        """Escalation never lands on a group whose window is unknown, so it cannot raise the
+        maximum, and a maximum over the rest would promise what the router may not serve."""
+        router = self._router(
+            _BIG,
+            ("unmapped", "openai/no-such-model-xyz", None),
+            tiers={"SIMPLE": "unmapped", "COMPLEX": "big-model"},
+        )
+        assert router.advertised_context_window() is None
+
+    def test_a_group_with_one_unresolvable_deployment_reports_nothing(self):
+        """The group's own minimum is only a floor over the members that resolved. The core router
+        can still pick the unmapped sibling, whose real window may be smaller than either."""
+        router = self._router(
+            ("partly-mapped", "openai/gpt-4o-mini", 200000),
+            ("partly-mapped", "openai/no-such-model-xyz", None),
+            tiers={"SIMPLE": "partly-mapped"},
+        )
+        assert router.advertised_context_window() is None
+
+    def test_a_tier_naming_a_group_that_does_not_exist_reports_nothing(self):
+        router = self._router(_SMALL, tiers={"SIMPLE": "small-model", "COMPLEX": "absent-group"})
+        assert router.advertised_context_window() is None

tests/test_litellm/test_router.py

@@ -15644,3 +15644,50 @@ async def test_an_open_circuit_breaker_skips_the_session_binding_without_a_warni
     assert binding is None
     assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
     assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)
+
+
+def _auto_router_deployment(model_name: str, tiers: dict, tags: list[str] | None = None) -> dict:
+    return {
+        "model_name": model_name,
+        "litellm_params": {
+            "model": "auto_router/complexity_router",
+            "complexity_router_config": {"tiers": tiers},
+            "complexity_router_default_model": "narrow-model",
+            **({"tags": tags} if tags is not None else {}),
+        },
+    }
+
+
+def _tiered_router(*auto_routers: dict) -> litellm.Router:
+    return litellm.Router(
+        model_list=[
+            {"model_name": "narrow-model", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "k"}},
+            {"model_name": "wide-model", "litellm_params": {"model": "openai/gpt-4.1", "api_key": "k"}},
+            *auto_routers,
+        ]
+    )
+
+
+def test_get_auto_router_context_window_derives_from_the_tier_groups():
+    router = _tiered_router(_auto_router_deployment("auto", {"SIMPLE": "narrow-model", "COMPLEX": "wide-model"}))
+
+    assert router.get_auto_router_context_window("auto") == 1_047_576
+
+
+def test_get_auto_router_context_window_is_none_for_a_name_no_auto_router_owns():
+    router = _tiered_router(_auto_router_deployment("auto", {"SIMPLE": "narrow-model"}))
+
+    assert router.get_auto_router_context_window("narrow-model") is None
+    assert router.get_auto_router_context_window("not-a-real-model") is None
+
+
+def test_get_auto_router_context_window_is_the_narrowest_strategy_tagged_under_one_name():
+    """Tags select one strategy rather than escalating between them, so a caller whose tags land on
+    the narrowest gets no second chance at a wider one."""
+    router = _tiered_router(
+        _auto_router_deployment("auto", {"SIMPLE": "narrow-model"}, tags=["default"]),
+        _auto_router_deployment("auto", {"SIMPLE": "wide-model"}, tags=["big"]),
+    )
+
+    assert len(router.complexity_routers["auto"]) == 2
+    assert router.get_auto_router_context_window("auto") == 128_000