This commit is contained in:
devin-ai-integration[bot] 2026-10-30 19:56:38 -07:00 • committed by GitHub
commit 53fd99d817
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 547 additions and 38 deletions

View file

@ -1257,7 +1257,7 @@ def stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str,
)
def _exact_base_model_max_input_tokens(base_model: str) -> int | None:
def exact_base_model_max_input_tokens(base_model: str) -> int | None:
try:
model_info: Final = litellm.get_model_info(model=base_model)
except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model
@ -1294,7 +1294,7 @@ def _base_model_widened_window(
return MappingProxyType({})
if not _names_route_model(base_model, route_model_info["key"], custom_llm_provider):
return MappingProxyType({})
base_max_input_tokens: Final = _exact_base_model_max_input_tokens(base_model)
base_max_input_tokens: Final = exact_base_model_max_input_tokens(base_model)
if base_max_input_tokens is None or base_max_input_tokens <= route_max_input_tokens:
return MappingProxyType({})
return MappingProxyType({"max_input_tokens": base_max_input_tokens})

View file

@ -1342,18 +1342,48 @@ def _window_can_hold(window: int | None, needed: int, buffer: float) -> bool:
return window is None or needed <= int(window * buffer)
def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool:
window, has_unknown = facts
return window is not None and not has_unknown and needed <= int(window * buffer)
def _declared_base_model(deployment: Mapping[str, object]) -> str | None:
match deployment:
case {"model_info": {"base_model": str() as base_model}} | {
"litellm_params": {"base_model": str() as base_model}
}:
return base_model
case _:
return None
def _inference_profile_base_model_window(deployment: Mapping[str, object]) -> int | None:
from litellm.llms.bedrock.common_utils import is_bedrock_application_inference_profile_arn
from litellm.router import exact_base_model_max_input_tokens
base_model: Final = _declared_base_model(deployment)
match deployment:
case {"litellm_params": {"model": str() as model}} if (
base_model is not None and is_bedrock_application_inference_profile_arn(model)
):
return exact_base_model_max_input_tokens(base_model)
case _:
return None
class _RequestContextFit(NamedTuple):
facts: Mapping[str, tuple[int | None, bool]]
needed: int | None
buffer: float
compaction_owned: frozenset[str] = frozenset()
def accepts(self, model: str) -> bool:
return self.needed is None or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer)
return (
self.needed is None
or model in self.compaction_owned
or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer)
)
def can_escalate_to(self, model: str, needed: int) -> bool:
window, has_unknown = self.facts.get(model, (None, True))
if window is None or has_unknown:
return False
return model in self.compaction_owned or needed <= int(window * self.buffer)
class _ContextWindowPlacement(NamedTuple):
@ -3502,7 +3532,15 @@ class ComplexityRouter(CustomLogger):
return name if self.config.has_custom_tiers else ComplexityTier(name)
def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None:
return self._deployment_limit(group, deployment, "max_input_tokens")
window: Final = self._deployment_limit(group, deployment, "max_input_tokens")
return window if window is not None else _inference_profile_base_model_window(deployment)
def _window_hidden_from_compaction(self, group: str) -> bool:
return any(
_inference_profile_base_model_window(deployment) is not None
and self._deployment_limit(group, deployment, "max_input_tokens") is None
for deployment in self._group_deployments(group)
)
def _deployment_limit(
self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"]
@ -3615,15 +3653,19 @@ class ComplexityRouter(CustomLogger):
resolved_messages: Sequence[Mapping[str, object]] | None,
request_kwargs: Mapping[str, object],
) -> _RequestContextFit:
if (
compaction_pending(request_kwargs)
or not self.config.enable_context_window_escalation
or not resolved_messages
):
if not self.config.enable_context_window_escalation or not resolved_messages:
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
(self.config.default_model,) if self.config.default_model else ()
)
compacting: Final = compaction_pending(request_kwargs)
compaction_owned: Final = (
frozenset(name for name in names if not self._window_hidden_from_compaction(name))
if compacting
else frozenset[str]()
)
if compacting and compaction_owned == names:
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
facts: Final = MappingProxyType({name: self._group_window_facts(name) for name in names})
known: Final = tuple(window for window, _ in facts.values() if window is not None)
buffer: Final = self.config.context_window_escalation_buffer
@ -3631,7 +3673,7 @@ class ComplexityRouter(CustomLogger):
min(known) * buffer
)
needed: Final = await self._counted_request_tokens(resolved_messages, request_kwargs) if needs_count else None
return _RequestContextFit(facts=facts, needed=needed, buffer=buffer)
return _RequestContextFit(facts=facts, needed=needed, buffer=buffer, compaction_owned=compaction_owned)
async def _context_window_placement(
self,
@ -3645,11 +3687,7 @@ class ComplexityRouter(CustomLogger):
(the placement stands). Only a real tokenizer count ever moves a request, escalation
lands only on groups whose every deployment declares a fitting window, and a group
with no resolvable window is never moved on faith in either direction."""
if (
compaction_pending(request_kwargs)
or not self.config.enable_context_window_escalation
or not resolved_messages
):
if not self.config.enable_context_window_escalation or not resolved_messages:
return None
pools: Final = self._tier_pools()
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))
@ -3658,7 +3696,7 @@ class ComplexityRouter(CustomLogger):
fit: Final = context_fit or await self._request_context_fit(resolved_messages, request_kwargs)
if fit.needed is None:
return None
return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, facts=fit.facts, needed=fit.needed)
return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, fit=fit, needed=fit.needed)
def _placement_for_tokens(
self,
@ -3666,29 +3704,17 @@ class ComplexityRouter(CustomLogger):
tier: ComplexityTier | str,
pool: tuple[str, ...],
pools: Mapping[str, list[str]],
facts: Mapping[str, tuple[int | None, bool]],
fit: _RequestContextFit,
needed: int,
) -> _ContextWindowPlacement | None:
buffer: Final = self.config.context_window_escalation_buffer
in_tier: Final = tuple(
group for group in pool if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer)
)
in_tier: Final = tuple(group for group in pool if fit.accepts(group))
if in_tier and len(in_tier) == len(pool):
return None
holdable: Final = frozenset(
group
for tier_pool in pools.values()
for group in tier_pool
if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer)
)
holdable: Final = frozenset(group for tier_pool in pools.values() for group in tier_pool if fit.accepts(group))
if in_tier:
return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable)
for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]:
proven = tuple(
group
for group in pools.get(name, ())
if _group_provably_fits(facts.get(group, (None, True)), needed, buffer)
)
proven = tuple(group for group in pools.get(name, ()) if fit.can_escalate_to(group, needed))
if proven:
return _ContextWindowPlacement(
tier=name if self.config.has_custom_tiers else ComplexityTier(name),
@ -4067,7 +4093,7 @@ class ComplexityRouter(CustomLogger):
if not context_recovery
or candidate_tier == decided_tier
or fit.needed is None
or _group_provably_fits(fit.facts.get(model, (None, True)), fit.needed, fit.buffer)
or fit.can_escalate_to(model, fit.needed)
)
candidates: Final = tuple(
peer

View file

@ -143,7 +143,7 @@ ignored_function_names = [
"_route_capability_pin", # Exercised through Decisions routing tests in test_router.py
"_deployment_serves_route", # Exercised through mixed-group Decisions tests in test_router.py
"_without_deployments_that_cannot_serve_route", # Exercised through Decisions routing tests in test_router.py
"_exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"_base_model_widened_window", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py

View file

@ -30,6 +30,9 @@
- {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"}
- {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"}
- {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"}
- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3510", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"}
- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"}
- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"}
- {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"}
- {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix; runs only on a stack with the prompt_caching pre-call check enabled (E2E_PROMPT_CACHING_STACK)"}
- {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"}

View file

@ -1438,6 +1438,8 @@ class ModelInfoBody(BaseModel):
allowed_fails: int | None = None
allowed_fails_policy: dict[str, int] | None = None
token_count_model: TokenCountModelBody | None = None
base_model: str | None = None
max_input_tokens: int | None = None
class ModelNewBody(BaseModel):

View file

@ -989,12 +989,13 @@ class ProxyClient:
# ---- LLM calls ------------------------------------------------------
@step("Send a /chat/completions request to {body.model}")
def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]:
def chat(self, key: str, body: ChatBody, timeout: float | None = None) -> Result[ChatResponse]:
return self.transport.post(
"/chat/completions",
headers=self.transport.bearer(key),
json=body,
response_type=ChatResponse,
timeout=timeout,
)
@step("Send a streaming /chat/completions request to {body.model}")

View file

@ -0,0 +1,240 @@
"""Live e2e pins for the complexity router's context-window escalation when a tier is a Bedrock
application inference profile ARN whose window is declared only through ``model_info.base_model``.
The ARN names no model, so before the router took the base_model's window it saw no window at all,
the escalation gate treated that as able to hold anything, and an oversized request scored SIMPLE
stayed on the 200k model and 400'd at Bedrock instead of moving to the 1M MEDIUM tier.
The explicit-window control declares ``max_input_tokens`` below its model's own row on a separate
backend model, since the router registers a deployment's ``model_info`` under the shared cost-map key
of its ``model:`` and a declared window on the ARN would leak to the base_model deployment. A short
request on the base_model router proves the gate only moves what does not fit.
"""
import os
from collections.abc import Iterator
from contextlib import ExitStack
from dataclasses import dataclass
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_http import unwrap
from e2e_metadata import Domain, Mode, Provider, Subject, meta
from lifecycle import ResourceManager
from models import (
ChatBody,
ChatMessage,
ChatResponse,
KeyGenerateBody,
LiteLLMParamsBody,
ModelInfoBody,
ModelNewBody,
SpendLogRow,
)
from proxy_client import ProxyClient
pytestmark = pytest.mark.e2e
SIMPLE_BACKEND: Final = os.environ.get(
"E2E_BEDROCK_INFERENCE_PROFILE_ARN",
"bedrock/converse/arn:aws:bedrock:us-west-2:888602223428:application-inference-profile/sccejsk6r44v",
)
SIMPLE_BASE_MODEL: Final = "global.anthropic.claude-haiku-4-5-20251001-v1:0"
EXPLICIT_WINDOW_BACKEND: Final = "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0"
DECLARED_WINDOW_TOKENS: Final = 60_000
UPPER_BACKEND: Final = "bedrock/converse/global.amazon.nova-2-lite-v1:0"
MAX_TOKENS: Final = 8
OVERSIZED_PROMPT_CHARS: Final = 1_000_000
BEYOND_DECLARED_WINDOW_PROMPT_CHARS: Final = 400_000
LONG_PROMPT_TIMEOUT_SECONDS: Final = 180.0
FILLER: Final = (
"On the morning of day {n}, the garden club met beside the old mill and talked about which flowers "
"to grow along the river walk. Everyone brought warm bread and shared stories about the cold winter "
"and the first robin of spring.\n"
)
ASK: Final = "How many days are written above? Reply with the single word OK."
@dataclass(frozen=True, slots=True)
class EscalationRouters:
base_model_router: str
explicit_window_router: str
simple_from_base_model: str
simple_with_explicit_window: str
upper: str
def _bedrock_params(model: str) -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=model,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="us-west-2",
)
def _router_params(simple: str, upper: str) -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model="auto_router/complexity_router",
complexity_router_config={
"classifier_type": "heuristic",
"enable_context_window_escalation": True,
"tiers": {"SIMPLE": simple, "MEDIUM": upper, "COMPLEX": upper, "REASONING": upper},
},
)
def _prompt_of(chars: int) -> str:
marker: Final = unique_marker()
line_count: Final = -(-chars // len(FILLER.format(n=f"1 ({marker})")))
lines: Final = tuple(FILLER.format(n=f"{n} ({marker})") for n in range(1, line_count + 1))
return "".join(lines) + "\n" + ASK
def _chat_body(model: str, content: str) -> ChatBody:
return ChatBody(model=model, messages=[ChatMessage(role="user", content=content)], max_tokens=MAX_TOKENS)
def _key_for(proxy: ProxyClient, resources: ResourceManager, models: list[str]) -> str:
key: Final = proxy.generate_key(KeyGenerateBody(models=models, user_id="e2e-context-window-base-model"))
resources.defer(lambda: proxy.delete_key(key))
return key
def _served_by(rows: list[SpendLogRow], alias: str, backend: str, context: str) -> None:
served: Final = tuple(row.model for row in rows)
assert served and all(model in {alias, backend} for model in served), (
f"{context}: expected every request to be served by {alias} ({backend}), spend logs show {served}"
)
def _answered(response: ChatResponse) -> None:
message: Final = response.choices[0].message if response.choices else None
assert message is not None and message.content, f"expected an answer, got {response}"
@pytest.fixture(scope="class")
def router_stack() -> Iterator[ExitStack]:
with ExitStack() as stack:
yield stack
@pytest.fixture(scope="class")
def routers(proxy: ProxyClient, router_stack: ExitStack) -> EscalationRouters:
marker: Final = unique_marker()
named: Final = EscalationRouters(
base_model_router=f"e2e-cw-router-base-{marker}",
explicit_window_router=f"e2e-cw-router-explicit-{marker}",
simple_from_base_model=f"e2e-cw-simple-base-{marker}",
simple_with_explicit_window=f"e2e-cw-simple-explicit-{marker}",
upper=f"e2e-cw-upper-{marker}",
)
registrations: Final = (
ModelNewBody(
model_name=named.simple_from_base_model,
litellm_params=_bedrock_params(SIMPLE_BACKEND),
model_info=ModelInfoBody(base_model=SIMPLE_BASE_MODEL),
),
ModelNewBody(
model_name=named.simple_with_explicit_window,
litellm_params=_bedrock_params(EXPLICIT_WINDOW_BACKEND),
model_info=ModelInfoBody(max_input_tokens=DECLARED_WINDOW_TOKENS),
),
ModelNewBody(model_name=named.upper, litellm_params=_bedrock_params(UPPER_BACKEND), model_info=ModelInfoBody()),
ModelNewBody(
model_name=named.base_model_router,
litellm_params=_router_params(named.simple_from_base_model, named.upper),
model_info=ModelInfoBody(),
),
ModelNewBody(
model_name=named.explicit_window_router,
litellm_params=_router_params(named.simple_with_explicit_window, named.upper),
model_info=ModelInfoBody(),
),
)
for body in registrations:
router_stack.callback(proxy.delete_model, proxy.register_model(body))
return named
class TestContextWindowEscalationFromBaseModel:
@pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_base_model_window")
@meta(
Subject(
domain=Domain.ROUTING,
providers=(Provider.BEDROCK,),
models=(SIMPLE_BACKEND, UPPER_BACKEND),
mode=Mode.NONSTREAM,
)
)
def test_base_model_window_escalates(
self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters
) -> None:
key: Final = _key_for(
proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper]
)
response: Final = unwrap(
proxy.chat(
key,
_chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS)),
timeout=LONG_PROMPT_TIMEOUT_SECONDS,
)
)
_answered(response)
_served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "oversized ask, window from base_model")
@pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_explicit_window")
@meta(
Subject(
domain=Domain.ROUTING,
providers=(Provider.BEDROCK,),
models=(EXPLICIT_WINDOW_BACKEND, UPPER_BACKEND),
mode=Mode.NONSTREAM,
)
)
def test_explicit_window_escalates(
self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters
) -> None:
key: Final = _key_for(
proxy, resources, [routers.explicit_window_router, routers.simple_with_explicit_window, routers.upper]
)
response: Final = unwrap(
proxy.chat(
key,
_chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS)),
timeout=LONG_PROMPT_TIMEOUT_SECONDS,
)
)
_answered(response)
_served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "ask beyond the declared window")
@pytest.mark.covers("reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier")
@meta(
Subject(
domain=Domain.ROUTING,
providers=(Provider.BEDROCK,),
models=(SIMPLE_BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_fitting_prompt_stays_on_the_simple_tier(
self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters
) -> None:
key: Final = _key_for(
proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper]
)
response: Final = unwrap(proxy.chat(key, _chat_body(routers.base_model_router, f"say hello {unique_marker()}")))
_answered(response)
_served_by(
proxy.poll_logs_for_key(key),
routers.simple_from_base_model,
SIMPLE_BACKEND,
"short ask, window from base_model",
)

View file

@ -313,6 +313,104 @@ async def test_undersized_native_compactor_does_not_block_explicit_escalation(
assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer"
@pytest.mark.parametrize("escalate", [False, True])
async def test_a_bedrock_arn_tier_windowed_only_through_base_model_is_never_compacted(
wire: tuple[respx.Route, respx.Route], escalate: bool,
) -> None:
base_model: Final = "meta.llama3-8b-instruct-v1:0"
base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"]
assert base_window is not None
router: Final = make_router(settings={
"enable_context_window_escalation": escalate,
"tiers": {"SIMPLE": "profile", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"},
})
router.add_deployment(Deployment(
model_name="profile", litellm_params={
"model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz",
"mock_response": "profile answer",
}, model_info={"id": "profile-answer", "base_model": base_model},
))
payload: Final = {"messages": [
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 14)},
{"role": "assistant", "content": "Recorded"},
{"role": "user", "content": "Answer with the project code"},
]}
output: Final = base_window // 4
trigger: Final = int(base_window * ContextCompactionConfig().trigger_ratio)
assert trigger - output < router._count_pre_call_check_tokens(payload["messages"], None, payload) < trigger
response: Final = await router.acompletion(model="auto", max_tokens=output, **payload)
assert all(route.call_count == 0 for route in wire)
assert response.choices[0].message.content == "profile answer"
async def test_an_armed_compactor_does_not_keep_an_oversized_history_on_an_inference_profile_tier(
wire: tuple[respx.Route, respx.Route],
) -> None:
base_model: Final = "meta.llama3-8b-instruct-v1:0"
base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"]
assert base_window is not None
router: Final = make_router(settings={
"enable_context_window_escalation": True,
"tiers": {"SIMPLE": "profile", "MEDIUM": "wide", "COMPLEX": "large", "REASONING": "large"},
})
router.add_deployment(Deployment(
model_name="profile", litellm_params={
"model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz",
"mock_response": "profile answer",
}, model_info={"id": "profile-answer", "base_model": base_model},
))
router.add_deployment(Deployment(
model_name="wide", litellm_params={
"model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test",
}, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64},
))
compactor, answer = wire
answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer"))
payload: Final = {"messages": [
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 7)},
{"role": "assistant", "content": "Recorded"},
{"role": "user", "content": "Answer with the project code"},
]}
counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload)
assert base_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64
response: Final = await invoke(router, "chat", payload)
assert compactor.call_count == 0 and answer.call_count == 1
assert "MAPLE-47" in str(response)
@pytest.mark.parametrize("chosen", ["SIMPLE", "MEDIUM"])
async def test_an_oversized_history_compacts_on_the_cheapest_compactable_tier_beside_an_inference_profile(
wire: tuple[respx.Route, respx.Route], chosen: str,
) -> None:
base_model: Final = "meta.llama3-8b-instruct-v1:0"
base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"]
assert base_window is not None
small_window: Final = base_window + base_window // 2
router: Final = make_router(small_window, settings={
"enable_context_window_escalation": True,
"keyword_tier_rules": [{"keywords": ["answer"], "tier": chosen}],
"tiers": {"SIMPLE": "profile", "MEDIUM": "small", "COMPLEX": "large", "REASONING": "large"},
})
router.add_deployment(Deployment(
model_name="profile", litellm_params={
"model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz",
"mock_response": "profile answer",
}, model_info={"id": "profile-answer", "base_model": base_model},
))
payload: Final = {"messages": [
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 5)},
{"role": "assistant", "content": "Recorded"},
{"role": "user", "content": "Answer with the project code"},
]}
counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload)
assert small_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64
response: Final = await invoke(router, "chat", payload)
compactor, answer = wire
assert compactor.call_count == 1 and answer.call_count == 1
assert "Background detail" not in answer.calls[0].request.content.decode()
assert "MAPLE-47" in str(response)
@pytest.mark.parametrize(
("surface", "failure"),
[(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")]

View file

@ -14786,6 +14786,24 @@ _CJK_TURNS = [
]
_BEDROCK_PROFILE_ARN: Final = (
"bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz"
)
_LLAMA_BASE_MODEL: Final = "meta.llama3-8b-instruct-v1:0"
def _turns_past(window: int) -> list[dict[str, str]]:
return [
{
"role": "user",
"content": "Here is everything discussed so far. "
+ "The meeting notes were saved to the shared folder for later review this week. " * (window // 4),
},
{"role": "assistant", "content": "Noted, I have read all of it."},
{"role": "user", "content": "ok continue"},
]
def _tier_config(**overrides: object) -> dict[str, object]:
return {
"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"},
@ -14825,6 +14843,127 @@ class TestContextWindowEscalation:
assert result.routing_decision["tier"] == "COMPLEX"
assert "context_escalation" in result.routing_decision["signals"]
@staticmethod
def _base_model_tier_router(
small: dict[str, object], base_window: int, **overrides: object
) -> ComplexityRouter:
litellm_router: Final = Router(
model_list=[
{"model_name": "small-model", **small},
{
"model_name": "big-model",
"litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"},
"model_info": {"max_input_tokens": base_window * 16},
},
]
)
return ComplexityRouter(
model_name="test-router",
litellm_router_instance=litellm_router,
complexity_router_config=_tier_config(**overrides),
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
"overrides",
[{}, {"adaptive": True, "session_affinity": True, "deployment_affinity": True}],
ids=["fixed", "adaptive"],
)
@pytest.mark.parametrize("declared_in", ["model_info", "litellm_params"])
async def test_an_inference_profile_window_declared_only_through_base_model_escalates_too(
self, overrides: dict[str, object], declared_in: str
):
base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"]
assert base_window is not None
router: Final = self._base_model_tier_router(
{
"litellm_params": {
"model": _BEDROCK_PROFILE_ARN,
"mock_response": "ok",
**({"base_model": _LLAMA_BASE_MODEL} if declared_in == "litellm_params" else {}),
},
"model_info": {"base_model": _LLAMA_BASE_MODEL} if declared_in == "model_info" else {},
},
base_window,
**overrides,
)
result: Final = await router.async_pre_routing_hook(
model="test-router",
request_kwargs={"metadata": {"session_id": "long-session"}},
messages=_turns_past(base_window),
)
assert result is not None
assert result.model == "big-model"
assert result.routing_decision["context_escalation_original_tier"] == "SIMPLE"
assert "context_escalation" in result.routing_decision["signals"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
("own_window", "model"),
[
("declared", _BEDROCK_PROFILE_ARN),
("registered", _BEDROCK_PROFILE_ARN),
(None, "openai/a-model-the-cost-map-does-not-know"),
(None, "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:provisioned-model/abc123xyz"),
(None, "bedrock/llama/arn:aws:bedrock:us-west-2:000000000000:imported-model/abc123xyz"),
],
ids=["declared", "registered", "not_a_bedrock_arn", "provisioned_model", "imported_model"],
)
async def test_base_model_never_overrides_an_own_window_or_reaches_a_non_profile_route(
self, own_window: str | None, model: str, monkeypatch: pytest.MonkeyPatch
):
base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"]
assert base_window is not None
if own_window == "registered":
monkeypatch.setitem(
litellm.model_cost,
_BEDROCK_PROFILE_ARN,
{"max_input_tokens": base_window * 8, "litellm_provider": "bedrock", "mode": "chat"},
)
router: Final = self._base_model_tier_router(
{
"litellm_params": {"model": model, "mock_response": "ok"},
"model_info": {
"base_model": _LLAMA_BASE_MODEL,
**({"max_input_tokens": base_window * 8} if own_window == "declared" else {}),
},
},
base_window,
)
result: Final = await router.async_pre_routing_hook(
model="test-router", request_kwargs={}, messages=_turns_past(base_window)
)
assert result is not None
assert result.model == "small-model"
assert "context_escalated" not in result.routing_decision
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_model_cost_map")
async def test_an_unmapped_base_model_leaves_a_bedrock_arn_tier_in_place(self):
base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"]
assert base_window is not None
router: Final = self._base_model_tier_router(
{
"litellm_params": {"model": _BEDROCK_PROFILE_ARN, "mock_response": "ok"},
"model_info": {"base_model": "anthropic.claude-not-in-the-cost-map-v1:0"},
},
base_window,
)
result: Final = await router.async_pre_routing_hook(
model="test-router", request_kwargs={}, messages=_turns_past(base_window)
)
assert result is not None
assert result.model == "small-model"
assert "context_escalated" not in result.routing_decision
@pytest.mark.asyncio
async def test_a_prompt_that_fits_routes_exactly_as_before(self):
"""The gate must be invisible for normal traffic: same model, no escalation facts."""