diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 606a590c24b..84008319cb0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9267,9 +9267,18 @@ def select_data_generator( ) -def get_litellm_model_info(model: dict = {}): +def get_litellm_model_info(model: dict = {}, llm_router: Router | None = None): model_info: Final = model.get("model_info", {}) model_to_lookup = model.get("litellm_params", {}).get("model", None) + if _is_auto_router_model(model): + window: Final = ( + llm_router.get_auto_router_context_window(str(model.get("model_name") or "")) + if llm_router is not None + else None + ) + if window is None: + return _EMPTY_MAPPING + return MappingProxyType({"max_input_tokens": window}) try: if "azure" in model_to_lookup or model_info.get("base_model"): model_to_lookup = model_info.get("base_model", None) @@ -13536,10 +13545,10 @@ def _enrich_model_info_with_litellm_data( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) + litellm_model_info = get_litellm_model_info(model=model, llm_router=llm_router) # 2nd pass on the model, try seeing if we can find model in litellm model_cost map - if litellm_model_info == {}: + if litellm_model_info == {} and not _is_auto_router_model(model): # use litellm_param model_name to get model_info litellm_params = model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) @@ -13548,7 +13557,7 @@ def _enrich_model_info_with_litellm_data( except Exception: litellm_model_info = {} # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map - if litellm_model_info == {}: + if litellm_model_info == {} and not _is_auto_router_model(model): # use litellm_param model_name to get model_info litellm_params = model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) @@ -14997,16 +15006,16 @@ def _translate_model_name_for_response(model: dict) -> dict: return {**model, "model_name": team_public} -def _get_proxy_model_info(model: dict) -> dict: +def _get_proxy_model_info(model: dict, llm_router: Router | None = None) -> dict: # provided model_info in config.yaml model_info: Final = model.get("model_info", {}) # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) + litellm_model_info = get_litellm_model_info(model=model, llm_router=llm_router) # 2nd pass on the model, try seeing if we can find model in litellm model_cost map - if litellm_model_info == {}: + if litellm_model_info == {} and not _is_auto_router_model(model): # use litellm_param model_name to get model_info litellm_params = model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) @@ -15015,7 +15024,7 @@ def _get_proxy_model_info(model: dict) -> dict: except Exception: litellm_model_info = {} # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map - if litellm_model_info == {}: + if litellm_model_info == {} and not _is_auto_router_model(model): # use litellm_param model_name to get model_info litellm_params = model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) @@ -15163,7 +15172,9 @@ async def model_info_v1( status_code=400, detail={"error": f"Model id = {litellm_model_id} not found on litellm proxy"}, ) - _deployment_info_dict = _get_proxy_model_info(model=deployment_info.model_dump(exclude_none=True)) + _deployment_info_dict = _get_proxy_model_info( + model=deployment_info.model_dump(exclude_none=True), llm_router=llm_router + ) single_model_list: list[dict] = [_deployment_info_dict] if prisma_client is not None: single_model_list = await _populate_team_access_on_models( diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..27206c45f3d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10266,15 +10266,39 @@ class Router: model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") for model_info, litellm_params in zip(model_infos, params) ) - if isinstance(key, str) and key + if isinstance(key, str) and key and not key.startswith(AUTO_ROUTER_MODEL_PREFIX) ) ) + configured_input: Final = self._widest_configured_limit(model_infos, "max_input_tokens") return DeploymentModelListingInfo( cost_map_keys=cost_map_keys, - max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), + max_input_tokens=( + configured_input if configured_input is not None else self.get_auto_router_context_window(model_name) + ), max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), ) + def get_auto_router_context_window(self, model_name: str) -> int | None: + """The context window an auto-router under ``model_name`` can serve, or None. + + An auto-router deployment is a marker whose ``litellm_params.model`` is absent from the cost + map, so every other source a listing consults answers nothing and the router advertises no + window at all. The window it can actually serve lives in the tier model groups it dispatches + to, and only the strategy knows those. + + Smallest across the strategies registered under the name: a listing carries no request + tags, tags select one strategy rather than escalating between them, and a caller whose tags + land on the narrowest gets no second chance at a wider one. + + Derived for complexity routers only; the adaptive, quality and semantic strategies hold + their candidates in a different shape and are not covered. + """ + tagged: Final = self.complexity_routers.get(model_name) or () + windows: Final = tuple( + window for window in (entry.strategy.advertised_context_window() for entry in tagged) if window is not None + ) + return min(windows) if windows else None + @staticmethod def _widest_configured_limit(model_infos: Sequence[Mapping[str, Any]], field: str) -> int | None: """The largest usable value of ``field`` across a group's configured model_info blocks.""" diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 9deccc9a468..d6bc003513c 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2615,6 +2615,29 @@ class ComplexityRouter(CustomLogger): deployments: Final = list_models(model_name=group) if callable(list_models) else None return tuple(deployments) if isinstance(deployments, list) else () + def advertised_context_window(self) -> int | None: + """The context window this router can serve, or None when any tier group's window is + unresolvable. + + A marker deployment carries no provider metadata, so a listing has nothing to report for it + unless the tiers are walked. The direction of each aggregate tracks whether a fit check + exists at that level. Within a group it is the smallest, because the core router picks a + deployment inside a group without one. Across the tier groups it is the widest only while + context-window escalation is on, since that is what moves a prompt off a tier too small to + hold it before dispatch; with escalation off the classifier's tier is final, so the only + window every request is sure of is the smallest. A group whose window is unknown is never + escalated onto and its presence makes the whole answer a guess: report nothing instead. + """ + groups: Final = frozenset(group for pool in self._tier_pools().values() for group in pool) + windows: Final = tuple( + window + for window, unknown in (self._group_window_facts(group) for group in groups) + if window is not None and not unknown + ) + if not groups or len(windows) != len(groups): + return None + return max(windows) if self.config.enable_context_window_escalation else min(windows) + def _group_output_ceiling(self, group: str) -> int | None: """Smallest max_output_tokens across the group's deployments, or None when any deployment declares none: the core router picks within the group without a fit check, and a ceiling diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index deb7289d2d1..ee4c6739a8b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -1073,3 +1073,40 @@ async def test_prometheus_fallback_stats_job_runs_when_the_lock_is_free_or_absen await jobs["prometheus_fallback_stats_job"]() assert send_fallback_stats.await_count == 2 + + +def _window_router(tiers: dict[str, str]): + import litellm + + return litellm.Router( + model_list=[ + {"model_name": "narrow", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "k"}}, + {"model_name": "wide", "litellm_params": {"model": "openai/gpt-4.1", "api_key": "k"}}, + { + "model_name": "router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": tiers}, + "complexity_router_default_model": "narrow", + }, + }, + ] + ) + + +def test_get_litellm_model_info_derives_window_for_an_auto_router_marker(): + router = _window_router({"SIMPLE": "narrow", "COMPLEX": "wide"}) + marker = {"model_name": "router", "litellm_params": {"model": "auto_router/complexity_router"}} + + assert get_litellm_model_info(model=marker).get("max_input_tokens") is None + assert get_litellm_model_info(model=marker, llm_router=router) == {"max_input_tokens": 1_047_576} + + +def test_get_litellm_model_info_still_prices_a_concrete_deployment_sharing_the_router_name(): + router = _window_router({"SIMPLE": "narrow", "COMPLEX": "wide"}) + concrete = {"model_name": "router", "litellm_params": {"model": "openai/gpt-4o-mini"}} + + result = get_litellm_model_info(model=concrete, llm_router=router) + + assert result.get("max_input_tokens") == 128_000 + assert result.get("input_cost_per_token") is not None diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py index 4c141bcf698..c86aac59eda 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py @@ -92,7 +92,7 @@ def configured_router(monkeypatch): monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", [{"model_name": "gpt-4"}]) monkeypatch.setattr(proxy_server, "user_model", None) - monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model: model) + monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model, llm_router=None: model) yield router diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index bc346874e0d..cffb39e1270 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -804,7 +804,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce monkeypatch.setattr(ps, "llm_model_list", [team_row]) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, llm_router=None: team_row) monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) caller = UserAPIKeyAuth( @@ -839,7 +839,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey monkeypatch.setattr(ps, "llm_model_list", [team_row]) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, llm_router=None: team_row) monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) monkeypatch.setattr(ps, "_filter_models_by_team_id", team_filter) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 9def21c0573..5334d8d8bb6 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -2328,3 +2328,118 @@ class TestPrismaClientTokenAuthBehindThePool: assert isinstance(client.db, RoutingPrismaWrapper) assert client.db.writer.iam_token_db_auth is True assert client.db.reader.iam_token_db_auth is True + + +def _auto_router_model_list( + tiers: dict[str, str | list[str]], + router_model_info: dict[str, object] | None = None, +) -> list[dict]: + return [ + { + "model_name": "narrow", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + }, + { + "model_name": "wide", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "test-key"}, + }, + { + "model_name": "unmapped", + "litellm_params": {"model": "openai/no-such-model-xyz", "api_key": "test-key"}, + }, + { + "model_name": "router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": tiers}, + "complexity_router_default_model": "narrow", + }, + **({"model_info": router_model_info} if router_model_info is not None else {}), + }, + ] + + +def test_create_model_info_response_derives_window_from_auto_router_tiers(): + router = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow", "COMPLEX": "wide"})) + + response = create_model_info_response( + model_id="router", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert response["max_input_tokens"] == 1_047_576 + + +def test_create_model_info_response_auto_router_window_is_widest_across_tiers(): + """The narrow tier must not decide the advertised window: escalation moves an oversized + prompt onto a tier that fits, so the wide end is what the router serves.""" + narrow_only = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow"})) + both = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow", "COMPLEX": "wide"})) + + def window(router: litellm.Router) -> int | None: + return create_model_info_response( + model_id="router", provider="openai", llm_router=router, get_model_info=_raise_unmapped + ).get("max_input_tokens") + + assert window(narrow_only) == 128_000 + assert window(both) == 1_047_576 + + +def test_create_model_info_response_omits_auto_router_window_when_a_tier_is_unresolvable(): + router = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "wide", "COMPLEX": "unmapped"})) + + response = create_model_info_response( + model_id="router", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert "max_input_tokens" not in response + + +def test_create_model_info_response_auto_router_configured_window_outranks_derivation(): + router = litellm.Router( + model_list=_auto_router_model_list( + {"SIMPLE": "narrow", "COMPLEX": "wide"}, router_model_info={"max_input_tokens": 555_000} + ) + ) + + response = create_model_info_response( + model_id="router", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert response["max_input_tokens"] == 555_000 + + +def test_create_model_info_response_auto_router_window_does_not_leak_between_routers(): + """Every auto-router marker shares one cost-map key, so a window declared on one router used to + come back as the advertised window of every other router in the process.""" + router = litellm.Router( + model_list=[ + *_auto_router_model_list({"SIMPLE": "wide", "COMPLEX": "unmapped"}), + { + "model_name": "declared-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "narrow"}}, + "complexity_router_default_model": "narrow", + }, + "model_info": {"max_input_tokens": 555_000}, + }, + ] + ) + + def window(model_id: str) -> int | None: + return create_model_info_response( + model_id=model_id, provider="openai", llm_router=router, get_model_info=litellm.get_model_info + ).get("max_input_tokens") + + assert litellm.model_cost.get("auto_router/complexity_router", {}).get("max_input_tokens") == 555_000 + assert window("declared-router") == 555_000 + assert window("router") is None diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index dba44d1e2e8..b8bee687a06 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -15184,3 +15184,67 @@ class TestNonReasoningTier: "complex", "reasoning", ) + + +class TestAdvertisedContextWindow: + """A marker deployment carries no provider metadata, so a listing reports no window for the + router unless the tier groups are walked. The direction of each aggregate tracks whether a fit + check exists at that level: escalation checks fit between tiers, nothing checks it within a + group. + """ + + @staticmethod + def _router(*deployments: tuple, **overrides) -> ComplexityRouter: + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(*deployments), + complexity_router_config=_tier_config(**overrides), + ) + + def test_reports_the_widest_tier_a_prompt_can_be_escalated_onto(self): + router = self._router(_SMALL, _BIG) + assert router.advertised_context_window() == 200000 + + def test_escalation_off_reports_the_tier_a_request_cannot_be_moved_off(self): + """With escalation disabled the classifier's tier is final, so the widest would promise a + window that a request classified onto the narrow tier can never reach.""" + router = self._router(_SMALL, _BIG, enable_context_window_escalation=False) + assert router.advertised_context_window() == 16385 + + def test_a_single_tier_router_reports_that_tier(self): + router = self._router(_SMALL, _BIG, tiers={"SIMPLE": "small-model"}) + assert router.advertised_context_window() == 16385 + + def test_a_group_is_judged_by_its_smallest_member(self): + """Two deployments serve one tier and the core router picks between them with no fit + check, so the group can only promise the smaller window.""" + router = self._router( + ("mixed", "openai/gpt-4o-mini", 200000), + ("mixed", "openai/gpt-3.5-turbo", 16385), + tiers={"SIMPLE": "mixed"}, + ) + assert router.advertised_context_window() == 16385 + + def test_an_unresolvable_tier_makes_the_whole_answer_a_guess(self): + """Escalation never lands on a group whose window is unknown, so it cannot raise the + maximum, and a maximum over the rest would promise what the router may not serve.""" + router = self._router( + _BIG, + ("unmapped", "openai/no-such-model-xyz", None), + tiers={"SIMPLE": "unmapped", "COMPLEX": "big-model"}, + ) + assert router.advertised_context_window() is None + + def test_a_group_with_one_unresolvable_deployment_reports_nothing(self): + """The group's own minimum is only a floor over the members that resolved. The core router + can still pick the unmapped sibling, whose real window may be smaller than either.""" + router = self._router( + ("partly-mapped", "openai/gpt-4o-mini", 200000), + ("partly-mapped", "openai/no-such-model-xyz", None), + tiers={"SIMPLE": "partly-mapped"}, + ) + assert router.advertised_context_window() is None + + def test_a_tier_naming_a_group_that_does_not_exist_reports_nothing(self): + router = self._router(_SMALL, tiers={"SIMPLE": "small-model", "COMPLEX": "absent-group"}) + assert router.advertised_context_window() is None diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index f5e9b2091a0..7fe9afa1445 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15644,3 +15644,50 @@ async def test_an_open_circuit_breaker_skips_the_session_binding_without_a_warni assert binding is None assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == [] assert any("circuit breaker is open" in record.getMessage() for record in caplog.records) + + +def _auto_router_deployment(model_name: str, tiers: dict, tags: list[str] | None = None) -> dict: + return { + "model_name": model_name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": tiers}, + "complexity_router_default_model": "narrow-model", + **({"tags": tags} if tags is not None else {}), + }, + } + + +def _tiered_router(*auto_routers: dict) -> litellm.Router: + return litellm.Router( + model_list=[ + {"model_name": "narrow-model", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "k"}}, + {"model_name": "wide-model", "litellm_params": {"model": "openai/gpt-4.1", "api_key": "k"}}, + *auto_routers, + ] + ) + + +def test_get_auto_router_context_window_derives_from_the_tier_groups(): + router = _tiered_router(_auto_router_deployment("auto", {"SIMPLE": "narrow-model", "COMPLEX": "wide-model"})) + + assert router.get_auto_router_context_window("auto") == 1_047_576 + + +def test_get_auto_router_context_window_is_none_for_a_name_no_auto_router_owns(): + router = _tiered_router(_auto_router_deployment("auto", {"SIMPLE": "narrow-model"})) + + assert router.get_auto_router_context_window("narrow-model") is None + assert router.get_auto_router_context_window("not-a-real-model") is None + + +def test_get_auto_router_context_window_is_the_narrowest_strategy_tagged_under_one_name(): + """Tags select one strategy rather than escalating between them, so a caller whose tags land on + the narrowest gets no second chance at a wider one.""" + router = _tiered_router( + _auto_router_deployment("auto", {"SIMPLE": "narrow-model"}, tags=["default"]), + _auto_router_deployment("auto", {"SIMPLE": "wide-model"}, tags=["big"]), + ) + + assert len(router.complexity_routers["auto"]) == 2 + assert router.get_auto_router_context_window("auto") == 128_000