fix(auto-router): advertise the context window an auto-router can serve

An auto-router deployment is a marker whose litellm_params.model is absent from
the cost map, so every source the model-info paths consult answers nothing and
the router advertises no context window at all. Clients that budget against the
advertised window then overflow and take an upstream context_length_exceeded 400
that no fallback catches, since the fallback map is keyed on model groups rather
than on the router.

Derive it from the tier model groups instead. Widest across the groups, because
context-window escalation moves a prompt off a tier that cannot hold it onto one
that can before dispatch, so the wide end is what the router serves; smallest
within each group, because the core router picks a deployment inside a group with
no such fit check. A group whose window is unresolvable is never escalated onto,
so it cannot raise the maximum and its presence makes the answer a guess: report
nothing. An operator-set model_info.max_input_tokens still wins everywhere.

Covers both model-info paths: get_model_listing_info for /v1/models, and
get_litellm_model_info for /model/info, /v1/model/info and /v2/model/info.
This commit is contained in:
Tin Chi Lo 2026-09-10 20:11:37 -07:00
parent 7419a536ad
commit 9b0ffd564c
10 changed files with 337 additions and 16 deletions

View file

@ -9175,9 +9175,18 @@ def select_data_generator(
)
def get_litellm_model_info(model: dict = {}):
def get_litellm_model_info(model: dict = {}, llm_router: Router | None = None):
model_info: Final = model.get("model_info", {})
model_to_lookup = model.get("litellm_params", {}).get("model", None)
if _is_auto_router_model(model):
window: Final = (
llm_router.get_auto_router_context_window(str(model.get("model_name") or ""))
if llm_router is not None
else None
)
if window is None:
return _EMPTY_MAPPING
return MappingProxyType({"max_input_tokens": window})
try:
if "azure" in model_to_lookup or model_info.get("base_model"):
model_to_lookup = model_info.get("base_model", None)
@ -13440,10 +13449,10 @@ def _enrich_model_info_with_litellm_data(
# read litellm model_prices_and_context_window.json to get the following:
# input_cost_per_token, output_cost_per_token, max_tokens
litellm_model_info = get_litellm_model_info(model=model)
litellm_model_info = get_litellm_model_info(model=model, llm_router=llm_router)
# 2nd pass on the model, try seeing if we can find model in litellm model_cost map
if litellm_model_info == {}:
if litellm_model_info == {} and not _is_auto_router_model(model):
# use litellm_param model_name to get model_info
litellm_params = model.get("litellm_params", {})
litellm_model = litellm_params.get("model", None)
@ -13452,7 +13461,7 @@ def _enrich_model_info_with_litellm_data(
except Exception:
litellm_model_info = {}
# 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map
if litellm_model_info == {}:
if litellm_model_info == {} and not _is_auto_router_model(model):
# use litellm_param model_name to get model_info
litellm_params = model.get("litellm_params", {})
litellm_model = litellm_params.get("model", None)
@ -14901,16 +14910,16 @@ def _translate_model_name_for_response(model: dict) -> dict:
return {**model, "model_name": team_public}
def _get_proxy_model_info(model: dict) -> dict:
def _get_proxy_model_info(model: dict, llm_router: Router | None = None) -> dict:
# provided model_info in config.yaml
model_info: Final = model.get("model_info", {})
# read litellm model_prices_and_context_window.json to get the following:
# input_cost_per_token, output_cost_per_token, max_tokens
litellm_model_info = get_litellm_model_info(model=model)
litellm_model_info = get_litellm_model_info(model=model, llm_router=llm_router)
# 2nd pass on the model, try seeing if we can find model in litellm model_cost map
if litellm_model_info == {}:
if litellm_model_info == {} and not _is_auto_router_model(model):
# use litellm_param model_name to get model_info
litellm_params = model.get("litellm_params", {})
litellm_model = litellm_params.get("model", None)
@ -14919,7 +14928,7 @@ def _get_proxy_model_info(model: dict) -> dict:
except Exception:
litellm_model_info = {}
# 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map
if litellm_model_info == {}:
if litellm_model_info == {} and not _is_auto_router_model(model):
# use litellm_param model_name to get model_info
litellm_params = model.get("litellm_params", {})
litellm_model = litellm_params.get("model", None)
@ -15067,7 +15076,9 @@ async def model_info_v1(
status_code=400,
detail={"error": f"Model id = {litellm_model_id} not found on litellm proxy"},
)
_deployment_info_dict = _get_proxy_model_info(model=deployment_info.model_dump(exclude_none=True))
_deployment_info_dict = _get_proxy_model_info(
model=deployment_info.model_dump(exclude_none=True), llm_router=llm_router
)
single_model_list: list[dict] = [_deployment_info_dict]
if prisma_client is not None:
single_model_list = await _populate_team_access_on_models(

View file

@ -10251,15 +10251,39 @@ class Router:
model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
for model_info, litellm_params in zip(model_infos, params)
)
if isinstance(key, str) and key
if isinstance(key, str) and key and not key.startswith(AUTO_ROUTER_MODEL_PREFIX)
)
)
configured_input: Final = self._widest_configured_limit(model_infos, "max_input_tokens")
return DeploymentModelListingInfo(
cost_map_keys=cost_map_keys,
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),
max_input_tokens=(
configured_input if configured_input is not None else self.get_auto_router_context_window(model_name)
),
max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
)
def get_auto_router_context_window(self, model_name: str) -> int | None:
"""The context window an auto-router under ``model_name`` can serve, or None.
An auto-router deployment is a marker whose ``litellm_params.model`` is absent from the cost
map, so every other source a listing consults answers nothing and the router advertises no
window at all. The window it can actually serve lives in the tier model groups it dispatches
to, and only the strategy knows those.
Smallest across the strategies registered under the name: a listing carries no request
tags, tags select one strategy rather than escalating between them, and a caller whose tags
land on the narrowest gets no second chance at a wider one.
Derived for complexity routers only; the adaptive, quality and semantic strategies hold
their candidates in a different shape and are not covered.
"""
tagged: Final = self.complexity_routers.get(model_name) or ()
windows: Final = tuple(
window for window in (entry.strategy.advertised_context_window() for entry in tagged) if window is not None
)
return min(windows) if windows else None
@staticmethod
def _widest_configured_limit(model_infos: Sequence[Mapping[str, Any]], field: str) -> int | None:
"""The largest usable value of ``field`` across a group's configured model_info blocks."""

View file

@ -2605,6 +2605,29 @@ class ComplexityRouter(CustomLogger):
deployments: Final = list_models(model_name=group) if callable(list_models) else None
return tuple(deployments) if isinstance(deployments, list) else ()
def advertised_context_window(self) -> int | None:
"""The context window this router can serve, or None when any tier group's window is
unresolvable.
A marker deployment carries no provider metadata, so a listing has nothing to report for it
unless the tiers are walked. The direction of each aggregate tracks whether a fit check
exists at that level. Within a group it is the smallest, because the core router picks a
deployment inside a group without one. Across the tier groups it is the widest only while
context-window escalation is on, since that is what moves a prompt off a tier too small to
hold it before dispatch; with escalation off the classifier's tier is final, so the only
window every request is sure of is the smallest. A group whose window is unknown is never
escalated onto and its presence makes the whole answer a guess: report nothing instead.
"""
groups: Final = frozenset(group for pool in self._tier_pools().values() for group in pool)
windows: Final = tuple(
window
for window, unknown in (self._group_window_facts(group) for group in groups)
if window is not None and not unknown
)
if not groups or len(windows) != len(groups):
return None
return max(windows) if self.config.enable_context_window_escalation else min(windows)
def _group_output_ceiling(self, group: str) -> int | None:
"""Smallest max_output_tokens across the group's deployments, or None when any deployment
declares none: the core router picks within the group without a fit check, and a ceiling

View file

@ -1073,3 +1073,40 @@ async def test_prometheus_fallback_stats_job_runs_when_the_lock_is_free_or_absen
await jobs["prometheus_fallback_stats_job"]()
assert send_fallback_stats.await_count == 2
def _window_router(tiers: dict[str, str]):
import litellm
return litellm.Router(
model_list=[
{"model_name": "narrow", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "k"}},
{"model_name": "wide", "litellm_params": {"model": "openai/gpt-4.1", "api_key": "k"}},
{
"model_name": "router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {"tiers": tiers},
"complexity_router_default_model": "narrow",
},
},
]
)
def test_get_litellm_model_info_derives_window_for_an_auto_router_marker():
router = _window_router({"SIMPLE": "narrow", "COMPLEX": "wide"})
marker = {"model_name": "router", "litellm_params": {"model": "auto_router/complexity_router"}}
assert get_litellm_model_info(model=marker).get("max_input_tokens") is None
assert get_litellm_model_info(model=marker, llm_router=router) == {"max_input_tokens": 1_047_576}
def test_get_litellm_model_info_still_prices_a_concrete_deployment_sharing_the_router_name():
router = _window_router({"SIMPLE": "narrow", "COMPLEX": "wide"})
concrete = {"model_name": "router", "litellm_params": {"model": "openai/gpt-4o-mini"}}
result = get_litellm_model_info(model=concrete, llm_router=router)
assert result.get("max_input_tokens") == 128_000
assert result.get("input_cost_per_token") is not None

View file

@ -92,7 +92,7 @@ def configured_router(monkeypatch):
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", [{"model_name": "gpt-4"}])
monkeypatch.setattr(proxy_server, "user_model", None)
monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model: model)
monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model, llm_router=None: model)
yield router

View file

@ -803,7 +803,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce
monkeypatch.setattr(ps, "llm_model_list", [team_row])
monkeypatch.setattr(ps, "llm_router", router)
monkeypatch.setattr(ps, "prisma_client", MagicMock())
monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row)
monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, llm_router=None: team_row)
monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate)
caller = UserAPIKeyAuth(
@ -838,7 +838,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey
monkeypatch.setattr(ps, "llm_model_list", [team_row])
monkeypatch.setattr(ps, "llm_router", router)
monkeypatch.setattr(ps, "prisma_client", MagicMock())
monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row)
monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, llm_router=None: team_row)
monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate)
monkeypatch.setattr(ps, "_filter_models_by_team_id", team_filter)

View file

@ -2254,3 +2254,118 @@ class TestPrismaClientTokenAuthBehindThePool:
assert isinstance(client.db, RoutingPrismaWrapper)
assert client.db.writer.iam_token_db_auth is True
assert client.db.reader.iam_token_db_auth is True
def _auto_router_model_list(
tiers: dict[str, str | list[str]],
router_model_info: dict[str, object] | None = None,
) -> list[dict]:
return [
{
"model_name": "narrow",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"},
},
{
"model_name": "wide",
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "test-key"},
},
{
"model_name": "unmapped",
"litellm_params": {"model": "openai/no-such-model-xyz", "api_key": "test-key"},
},
{
"model_name": "router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {"tiers": tiers},
"complexity_router_default_model": "narrow",
},
**({"model_info": router_model_info} if router_model_info is not None else {}),
},
]
def test_create_model_info_response_derives_window_from_auto_router_tiers():
router = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow", "COMPLEX": "wide"}))
response = create_model_info_response(
model_id="router",
provider="openai",
llm_router=router,
get_model_info=_raise_unmapped,
)
assert response["max_input_tokens"] == 1_047_576
def test_create_model_info_response_auto_router_window_is_widest_across_tiers():
"""The narrow tier must not decide the advertised window: escalation moves an oversized
prompt onto a tier that fits, so the wide end is what the router serves."""
narrow_only = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow"}))
both = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "narrow", "COMPLEX": "wide"}))
def window(router: litellm.Router) -> int | None:
return create_model_info_response(
model_id="router", provider="openai", llm_router=router, get_model_info=_raise_unmapped
).get("max_input_tokens")
assert window(narrow_only) == 128_000
assert window(both) == 1_047_576
def test_create_model_info_response_omits_auto_router_window_when_a_tier_is_unresolvable():
router = litellm.Router(model_list=_auto_router_model_list({"SIMPLE": "wide", "COMPLEX": "unmapped"}))
response = create_model_info_response(
model_id="router",
provider="openai",
llm_router=router,
get_model_info=_raise_unmapped,
)
assert "max_input_tokens" not in response
def test_create_model_info_response_auto_router_configured_window_outranks_derivation():
router = litellm.Router(
model_list=_auto_router_model_list(
{"SIMPLE": "narrow", "COMPLEX": "wide"}, router_model_info={"max_input_tokens": 555_000}
)
)
response = create_model_info_response(
model_id="router",
provider="openai",
llm_router=router,
get_model_info=_raise_unmapped,
)
assert response["max_input_tokens"] == 555_000
def test_create_model_info_response_auto_router_window_does_not_leak_between_routers():
"""Every auto-router marker shares one cost-map key, so a window declared on one router used to
come back as the advertised window of every other router in the process."""
router = litellm.Router(
model_list=[
*_auto_router_model_list({"SIMPLE": "wide", "COMPLEX": "unmapped"}),
{
"model_name": "declared-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {"tiers": {"SIMPLE": "narrow"}},
"complexity_router_default_model": "narrow",
},
"model_info": {"max_input_tokens": 555_000},
},
]
)
def window(model_id: str) -> int | None:
return create_model_info_response(
model_id=model_id, provider="openai", llm_router=router, get_model_info=litellm.get_model_info
).get("max_input_tokens")
assert litellm.model_cost.get("auto_router/complexity_router", {}).get("max_input_tokens") == 555_000
assert window("declared-router") == 555_000
assert window("router") is None

View file

@ -14608,3 +14608,67 @@ class TestNonReasoningTier:
"complex",
"reasoning",
)
class TestAdvertisedContextWindow:
"""A marker deployment carries no provider metadata, so a listing reports no window for the
router unless the tier groups are walked. The direction of each aggregate tracks whether a fit
check exists at that level: escalation checks fit between tiers, nothing checks it within a
group.
"""
@staticmethod
def _router(*deployments: tuple, **overrides) -> ComplexityRouter:
return ComplexityRouter(
model_name="test-router",
litellm_router_instance=_windowed_router(*deployments),
complexity_router_config=_tier_config(**overrides),
)
def test_reports_the_widest_tier_a_prompt_can_be_escalated_onto(self):
router = self._router(_SMALL, _BIG)
assert router.advertised_context_window() == 200000
def test_escalation_off_reports_the_tier_a_request_cannot_be_moved_off(self):
"""With escalation disabled the classifier's tier is final, so the widest would promise a
window that a request classified onto the narrow tier can never reach."""
router = self._router(_SMALL, _BIG, enable_context_window_escalation=False)
assert router.advertised_context_window() == 16385
def test_a_single_tier_router_reports_that_tier(self):
router = self._router(_SMALL, _BIG, tiers={"SIMPLE": "small-model"})
assert router.advertised_context_window() == 16385
def test_a_group_is_judged_by_its_smallest_member(self):
"""Two deployments serve one tier and the core router picks between them with no fit
check, so the group can only promise the smaller window."""
router = self._router(
("mixed", "openai/gpt-4o-mini", 200000),
("mixed", "openai/gpt-3.5-turbo", 16385),
tiers={"SIMPLE": "mixed"},
)
assert router.advertised_context_window() == 16385
def test_an_unresolvable_tier_makes_the_whole_answer_a_guess(self):
"""Escalation never lands on a group whose window is unknown, so it cannot raise the
maximum, and a maximum over the rest would promise what the router may not serve."""
router = self._router(
_BIG,
("unmapped", "openai/no-such-model-xyz", None),
tiers={"SIMPLE": "unmapped", "COMPLEX": "big-model"},
)
assert router.advertised_context_window() is None
def test_a_group_with_one_unresolvable_deployment_reports_nothing(self):
"""The group's own minimum is only a floor over the members that resolved. The core router
can still pick the unmapped sibling, whose real window may be smaller than either."""
router = self._router(
("partly-mapped", "openai/gpt-4o-mini", 200000),
("partly-mapped", "openai/no-such-model-xyz", None),
tiers={"SIMPLE": "partly-mapped"},
)
assert router.advertised_context_window() is None
def test_a_tier_naming_a_group_that_does_not_exist_reports_nothing(self):
router = self._router(_SMALL, tiers={"SIMPLE": "small-model", "COMPLEX": "absent-group"})
assert router.advertised_context_window() is None

View file

@ -15390,3 +15390,50 @@ async def test_an_open_circuit_breaker_skips_the_session_binding_without_a_warni
assert binding is None
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)
def _auto_router_deployment(model_name: str, tiers: dict, tags: list[str] | None = None) -> dict:
return {
"model_name": model_name,
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {"tiers": tiers},
"complexity_router_default_model": "narrow-model",
**({"tags": tags} if tags is not None else {}),
},
}
def _tiered_router(*auto_routers: dict) -> litellm.Router:
return litellm.Router(
model_list=[
{"model_name": "narrow-model", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "k"}},
{"model_name": "wide-model", "litellm_params": {"model": "openai/gpt-4.1", "api_key": "k"}},
*auto_routers,
]
)
def test_get_auto_router_context_window_derives_from_the_tier_groups():
router = _tiered_router(_auto_router_deployment("auto", {"SIMPLE": "narrow-model", "COMPLEX": "wide-model"}))
assert router.get_auto_router_context_window("auto") == 1_047_576
def test_get_auto_router_context_window_is_none_for_a_name_no_auto_router_owns():
router = _tiered_router(_auto_router_deployment("auto", {"SIMPLE": "narrow-model"}))
assert router.get_auto_router_context_window("narrow-model") is None
assert router.get_auto_router_context_window("not-a-real-model") is None
def test_get_auto_router_context_window_is_the_narrowest_strategy_tagged_under_one_name():
"""Tags select one strategy rather than escalating between them, so a caller whose tags land on
the narrowest gets no second chance at a wider one."""
router = _tiered_router(
_auto_router_deployment("auto", {"SIMPLE": "narrow-model"}, tags=["default"]),
_auto_router_deployment("auto", {"SIMPLE": "wide-model"}, tags=["big"]),
)
assert len(router.complexity_routers["auto"]) == 2
assert router.get_auto_router_context_window("auto") == 128_000

View file

@ -35064,7 +35064,7 @@ export interface components {
classification_prompt?: string | null;
/**
* Classifier Context Budget Chars
* @description Maximum characters of prior-turn text quoted to the LLM classifier, across the whole context window, per classification call. Turns are taken newest first and quoted whole while they fit, so a conversation small enough to quote entirely is never cut; once the budget runs out the older turns are dropped whole and only the turn straddling the boundary is truncated, into whatever space is left. The current ask and the caller's system prompt sit outside this budget and are always sent in full, as does the numbering each quoted turn carries. A budget under 120 leaves no room to quote a turn and suppresses the block; set classifier_context_window_size to 0 to turn context off deliberately. Only applies when classifier_type is 'llm'.
* @description Maximum characters of prior-turn text quoted to the LLM classifier, across the whole context window, per classification call. Turns are taken newest first and quoted whole while they fit, so a conversation small enough to quote entirely is never cut; once the budget runs out the older turns are dropped whole and only the turn straddling the boundary is truncated, into whatever space is left. The current ask and, except for Claude Code requests, the extracted system-role text sit outside this budget and are sent in full, as does the numbering each quoted turn carries. A budget under 120 leaves no room to quote a turn and suppresses the block; set classifier_context_window_size to 0 to turn context off deliberately. Only applies when classifier_type is 'llm'.
* @default 8000
*/
classifier_context_budget_chars: number;
@ -35081,7 +35081,7 @@ export interface components {
classifier_context_per_turn_chars?: number | null;
/**
* Classifier Context Window Size
* @description Number of prior user turns (tool output and harness reminders excluded) to include as context in the LLM classifier prompt, so a follow-up like 'now do the same for the streaming path' is classified against what it refers to. Counts turns of both roles when classifier_context_include_assistant_turns is enabled. These turns are sent to the classifier model, which may be a different deployment or provider than the routed completion model; that call already carries the current user ask and the caller's system prompt in full. Set to 0 to send neither prior turns nor any conversation context beyond the current ask. Only applies when classifier_type is 'llm'.
* @description Number of prior user turns (tool output and harness reminders excluded) to include as context in the LLM classifier prompt, so a follow-up like 'now do the same for the streaming path' is classified against what it refers to. Counts turns of both roles when classifier_context_include_assistant_turns is enabled. These turns are sent to the classifier model, which may be a different deployment or provider than the routed completion model; that call carries the current user ask and, except for Claude Code requests, the extracted system-role text in full. Claude Code system text is omitted to avoid classifying harness instructions; the routed completion still receives it. Set to 0 to send neither prior turns nor any conversation context beyond the current ask. Only applies when classifier_type is 'llm'.
* @default 3
*/
classifier_context_window_size: number;