diff --git a/litellm/router.py b/litellm/router.py index 535072d5fcf..516e6f5c879 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1257,7 +1257,7 @@ def stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str, ) -def _exact_base_model_max_input_tokens(base_model: str) -> int | None: +def exact_base_model_max_input_tokens(base_model: str) -> int | None: try: model_info: Final = litellm.get_model_info(model=base_model) except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model @@ -1294,7 +1294,7 @@ def _base_model_widened_window( return MappingProxyType({}) if not _names_route_model(base_model, route_model_info["key"], custom_llm_provider): return MappingProxyType({}) - base_max_input_tokens: Final = _exact_base_model_max_input_tokens(base_model) + base_max_input_tokens: Final = exact_base_model_max_input_tokens(base_model) if base_max_input_tokens is None or base_max_input_tokens <= route_max_input_tokens: return MappingProxyType({}) return MappingProxyType({"max_input_tokens": base_max_input_tokens}) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4f4ed76c170..b1e546cdb16 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1342,18 +1342,48 @@ def _window_can_hold(window: int | None, needed: int, buffer: float) -> bool: return window is None or needed <= int(window * buffer) -def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool: - window, has_unknown = facts - return window is not None and not has_unknown and needed <= int(window * buffer) +def _declared_base_model(deployment: Mapping[str, object]) -> str | None: + match deployment: + case {"model_info": {"base_model": str() as base_model}} | { + "litellm_params": {"base_model": str() as base_model} + }: + return base_model + case _: + return None + + +def _inference_profile_base_model_window(deployment: Mapping[str, object]) -> int | None: + from litellm.llms.bedrock.common_utils import is_bedrock_application_inference_profile_arn + from litellm.router import exact_base_model_max_input_tokens + + base_model: Final = _declared_base_model(deployment) + match deployment: + case {"litellm_params": {"model": str() as model}} if ( + base_model is not None and is_bedrock_application_inference_profile_arn(model) + ): + return exact_base_model_max_input_tokens(base_model) + case _: + return None class _RequestContextFit(NamedTuple): facts: Mapping[str, tuple[int | None, bool]] needed: int | None buffer: float + compaction_owned: frozenset[str] = frozenset() def accepts(self, model: str) -> bool: - return self.needed is None or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer) + return ( + self.needed is None + or model in self.compaction_owned + or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer) + ) + + def can_escalate_to(self, model: str, needed: int) -> bool: + window, has_unknown = self.facts.get(model, (None, True)) + if window is None or has_unknown: + return False + return model in self.compaction_owned or needed <= int(window * self.buffer) class _ContextWindowPlacement(NamedTuple): @@ -3502,7 +3532,15 @@ class ComplexityRouter(CustomLogger): return name if self.config.has_custom_tiers else ComplexityTier(name) def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None: - return self._deployment_limit(group, deployment, "max_input_tokens") + window: Final = self._deployment_limit(group, deployment, "max_input_tokens") + return window if window is not None else _inference_profile_base_model_window(deployment) + + def _window_hidden_from_compaction(self, group: str) -> bool: + return any( + _inference_profile_base_model_window(deployment) is not None + and self._deployment_limit(group, deployment, "max_input_tokens") is None + for deployment in self._group_deployments(group) + ) def _deployment_limit( self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"] @@ -3615,15 +3653,19 @@ class ComplexityRouter(CustomLogger): resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object], ) -> _RequestContextFit: - if ( - compaction_pending(request_kwargs) - or not self.config.enable_context_window_escalation - or not resolved_messages - ): + if not self.config.enable_context_window_escalation or not resolved_messages: return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset( (self.config.default_model,) if self.config.default_model else () ) + compacting: Final = compaction_pending(request_kwargs) + compaction_owned: Final = ( + frozenset(name for name in names if not self._window_hidden_from_compaction(name)) + if compacting + else frozenset[str]() + ) + if compacting and compaction_owned == names: + return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) facts: Final = MappingProxyType({name: self._group_window_facts(name) for name in names}) known: Final = tuple(window for window, _ in facts.values() if window is not None) buffer: Final = self.config.context_window_escalation_buffer @@ -3631,7 +3673,7 @@ class ComplexityRouter(CustomLogger): min(known) * buffer ) needed: Final = await self._counted_request_tokens(resolved_messages, request_kwargs) if needs_count else None - return _RequestContextFit(facts=facts, needed=needed, buffer=buffer) + return _RequestContextFit(facts=facts, needed=needed, buffer=buffer, compaction_owned=compaction_owned) async def _context_window_placement( self, @@ -3645,11 +3687,7 @@ class ComplexityRouter(CustomLogger): (the placement stands). Only a real tokenizer count ever moves a request, escalation lands only on groups whose every deployment declares a fitting window, and a group with no resolvable window is never moved on faith in either direction.""" - if ( - compaction_pending(request_kwargs) - or not self.config.enable_context_window_escalation - or not resolved_messages - ): + if not self.config.enable_context_window_escalation or not resolved_messages: return None pools: Final = self._tier_pools() pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ())) @@ -3658,7 +3696,7 @@ class ComplexityRouter(CustomLogger): fit: Final = context_fit or await self._request_context_fit(resolved_messages, request_kwargs) if fit.needed is None: return None - return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, facts=fit.facts, needed=fit.needed) + return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, fit=fit, needed=fit.needed) def _placement_for_tokens( self, @@ -3666,29 +3704,17 @@ class ComplexityRouter(CustomLogger): tier: ComplexityTier | str, pool: tuple[str, ...], pools: Mapping[str, list[str]], - facts: Mapping[str, tuple[int | None, bool]], + fit: _RequestContextFit, needed: int, ) -> _ContextWindowPlacement | None: - buffer: Final = self.config.context_window_escalation_buffer - in_tier: Final = tuple( - group for group in pool if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer) - ) + in_tier: Final = tuple(group for group in pool if fit.accepts(group)) if in_tier and len(in_tier) == len(pool): return None - holdable: Final = frozenset( - group - for tier_pool in pools.values() - for group in tier_pool - if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer) - ) + holdable: Final = frozenset(group for tier_pool in pools.values() for group in tier_pool if fit.accepts(group)) if in_tier: return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable) for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]: - proven = tuple( - group - for group in pools.get(name, ()) - if _group_provably_fits(facts.get(group, (None, True)), needed, buffer) - ) + proven = tuple(group for group in pools.get(name, ()) if fit.can_escalate_to(group, needed)) if proven: return _ContextWindowPlacement( tier=name if self.config.has_custom_tiers else ComplexityTier(name), @@ -4067,7 +4093,7 @@ class ComplexityRouter(CustomLogger): if not context_recovery or candidate_tier == decided_tier or fit.needed is None - or _group_provably_fits(fit.facts.get(model, (None, True)), fit.needed, fit.buffer) + or fit.can_escalate_to(model, fit.needed) ) candidates: Final = tuple( peer diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 69ad0c91ee9..c98366d0390 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -143,7 +143,7 @@ ignored_function_names = [ "_route_capability_pin", # Exercised through Decisions routing tests in test_router.py "_deployment_serves_route", # Exercised through mixed-group Decisions tests in test_router.py "_without_deployments_that_cannot_serve_route", # Exercised through Decisions routing tests in test_router.py - "_exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py + "exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_base_model_widened_window", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 7382cc215f2..bb55c10edb8 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -30,6 +30,9 @@ - {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"} - {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"} - {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"} +- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3510", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} +- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} +- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix; runs only on a stack with the prompt_caching pre-call check enabled (E2E_PROMPT_CACHING_STACK)"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 504adec5401..b3d3560766d 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1438,6 +1438,8 @@ class ModelInfoBody(BaseModel): allowed_fails: int | None = None allowed_fails_policy: dict[str, int] | None = None token_count_model: TokenCountModelBody | None = None + base_model: str | None = None + max_input_tokens: int | None = None class ModelNewBody(BaseModel): diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 8cc7bf0c2b4..f83459c6d76 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -989,12 +989,13 @@ class ProxyClient: # ---- LLM calls ------------------------------------------------------ @step("Send a /chat/completions request to {body.model}") - def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]: + def chat(self, key: str, body: ChatBody, timeout: float | None = None) -> Result[ChatResponse]: return self.transport.post( "/chat/completions", headers=self.transport.bearer(key), json=body, response_type=ChatResponse, + timeout=timeout, ) @step("Send a streaming /chat/completions request to {body.model}") diff --git a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py new file mode 100644 index 00000000000..c7c23edfc8b --- /dev/null +++ b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py @@ -0,0 +1,240 @@ +"""Live e2e pins for the complexity router's context-window escalation when a tier is a Bedrock +application inference profile ARN whose window is declared only through ``model_info.base_model``. + +The ARN names no model, so before the router took the base_model's window it saw no window at all, +the escalation gate treated that as able to hold anything, and an oversized request scored SIMPLE +stayed on the 200k model and 400'd at Bedrock instead of moving to the 1M MEDIUM tier. + +The explicit-window control declares ``max_input_tokens`` below its model's own row on a separate +backend model, since the router registers a deployment's ``model_info`` under the shared cost-map key +of its ``model:`` and a declared window on the ARN would leak to the base_model deployment. A short +request on the base_model router proves the gate only moves what does not fit. +""" + +import os +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Subject, meta +from lifecycle import ResourceManager +from models import ( + ChatBody, + ChatMessage, + ChatResponse, + KeyGenerateBody, + LiteLLMParamsBody, + ModelInfoBody, + ModelNewBody, + SpendLogRow, +) +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + +SIMPLE_BACKEND: Final = os.environ.get( + "E2E_BEDROCK_INFERENCE_PROFILE_ARN", + "bedrock/converse/arn:aws:bedrock:us-west-2:888602223428:application-inference-profile/sccejsk6r44v", +) +SIMPLE_BASE_MODEL: Final = "global.anthropic.claude-haiku-4-5-20251001-v1:0" +EXPLICIT_WINDOW_BACKEND: Final = "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" +DECLARED_WINDOW_TOKENS: Final = 60_000 +UPPER_BACKEND: Final = "bedrock/converse/global.amazon.nova-2-lite-v1:0" +MAX_TOKENS: Final = 8 +OVERSIZED_PROMPT_CHARS: Final = 1_000_000 +BEYOND_DECLARED_WINDOW_PROMPT_CHARS: Final = 400_000 +LONG_PROMPT_TIMEOUT_SECONDS: Final = 180.0 +FILLER: Final = ( + "On the morning of day {n}, the garden club met beside the old mill and talked about which flowers " + "to grow along the river walk. Everyone brought warm bread and shared stories about the cold winter " + "and the first robin of spring.\n" +) +ASK: Final = "How many days are written above? Reply with the single word OK." + + +@dataclass(frozen=True, slots=True) +class EscalationRouters: + base_model_router: str + explicit_window_router: str + simple_from_base_model: str + simple_with_explicit_window: str + upper: str + + +def _bedrock_params(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=model, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="us-west-2", + ) + + +def _router_params(simple: str, upper: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "heuristic", + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": simple, "MEDIUM": upper, "COMPLEX": upper, "REASONING": upper}, + }, + ) + + +def _prompt_of(chars: int) -> str: + marker: Final = unique_marker() + line_count: Final = -(-chars // len(FILLER.format(n=f"1 ({marker})"))) + lines: Final = tuple(FILLER.format(n=f"{n} ({marker})") for n in range(1, line_count + 1)) + return "".join(lines) + "\n" + ASK + + +def _chat_body(model: str, content: str) -> ChatBody: + return ChatBody(model=model, messages=[ChatMessage(role="user", content=content)], max_tokens=MAX_TOKENS) + + +def _key_for(proxy: ProxyClient, resources: ResourceManager, models: list[str]) -> str: + key: Final = proxy.generate_key(KeyGenerateBody(models=models, user_id="e2e-context-window-base-model")) + resources.defer(lambda: proxy.delete_key(key)) + return key + + +def _served_by(rows: list[SpendLogRow], alias: str, backend: str, context: str) -> None: + served: Final = tuple(row.model for row in rows) + assert served and all(model in {alias, backend} for model in served), ( + f"{context}: expected every request to be served by {alias} ({backend}), spend logs show {served}" + ) + + +def _answered(response: ChatResponse) -> None: + message: Final = response.choices[0].message if response.choices else None + assert message is not None and message.content, f"expected an answer, got {response}" + + +@pytest.fixture(scope="class") +def router_stack() -> Iterator[ExitStack]: + with ExitStack() as stack: + yield stack + + +@pytest.fixture(scope="class") +def routers(proxy: ProxyClient, router_stack: ExitStack) -> EscalationRouters: + marker: Final = unique_marker() + named: Final = EscalationRouters( + base_model_router=f"e2e-cw-router-base-{marker}", + explicit_window_router=f"e2e-cw-router-explicit-{marker}", + simple_from_base_model=f"e2e-cw-simple-base-{marker}", + simple_with_explicit_window=f"e2e-cw-simple-explicit-{marker}", + upper=f"e2e-cw-upper-{marker}", + ) + registrations: Final = ( + ModelNewBody( + model_name=named.simple_from_base_model, + litellm_params=_bedrock_params(SIMPLE_BACKEND), + model_info=ModelInfoBody(base_model=SIMPLE_BASE_MODEL), + ), + ModelNewBody( + model_name=named.simple_with_explicit_window, + litellm_params=_bedrock_params(EXPLICIT_WINDOW_BACKEND), + model_info=ModelInfoBody(max_input_tokens=DECLARED_WINDOW_TOKENS), + ), + ModelNewBody(model_name=named.upper, litellm_params=_bedrock_params(UPPER_BACKEND), model_info=ModelInfoBody()), + ModelNewBody( + model_name=named.base_model_router, + litellm_params=_router_params(named.simple_from_base_model, named.upper), + model_info=ModelInfoBody(), + ), + ModelNewBody( + model_name=named.explicit_window_router, + litellm_params=_router_params(named.simple_with_explicit_window, named.upper), + model_info=ModelInfoBody(), + ), + ) + for body in registrations: + router_stack.callback(proxy.delete_model, proxy.register_model(body)) + return named + + +class TestContextWindowEscalationFromBaseModel: + @pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_base_model_window") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.BEDROCK,), + models=(SIMPLE_BACKEND, UPPER_BACKEND), + mode=Mode.NONSTREAM, + ) + ) + def test_base_model_window_escalates( + self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters + ) -> None: + key: Final = _key_for( + proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper] + ) + + response: Final = unwrap( + proxy.chat( + key, + _chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS)), + timeout=LONG_PROMPT_TIMEOUT_SECONDS, + ) + ) + + _answered(response) + _served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "oversized ask, window from base_model") + + @pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_explicit_window") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.BEDROCK,), + models=(EXPLICIT_WINDOW_BACKEND, UPPER_BACKEND), + mode=Mode.NONSTREAM, + ) + ) + def test_explicit_window_escalates( + self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters + ) -> None: + key: Final = _key_for( + proxy, resources, [routers.explicit_window_router, routers.simple_with_explicit_window, routers.upper] + ) + + response: Final = unwrap( + proxy.chat( + key, + _chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS)), + timeout=LONG_PROMPT_TIMEOUT_SECONDS, + ) + ) + + _answered(response) + _served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "ask beyond the declared window") + + @pytest.mark.covers("reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.BEDROCK,), + models=(SIMPLE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_fitting_prompt_stays_on_the_simple_tier( + self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters + ) -> None: + key: Final = _key_for( + proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper] + ) + + response: Final = unwrap(proxy.chat(key, _chat_body(routers.base_model_router, f"say hello {unique_marker()}"))) + + _answered(response) + _served_by( + proxy.poll_logs_for_key(key), + routers.simple_from_base_model, + SIMPLE_BACKEND, + "short ask, window from base_model", + ) diff --git a/tests/unit/router_strategy/complexity_router/test_context_compaction.py b/tests/unit/router_strategy/complexity_router/test_context_compaction.py index 850accf574b..14850ba9a79 100644 --- a/tests/unit/router_strategy/complexity_router/test_context_compaction.py +++ b/tests/unit/router_strategy/complexity_router/test_context_compaction.py @@ -313,6 +313,104 @@ async def test_undersized_native_compactor_does_not_block_explicit_escalation( assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer" +@pytest.mark.parametrize("escalate", [False, True]) +async def test_a_bedrock_arn_tier_windowed_only_through_base_model_is_never_compacted( + wire: tuple[respx.Route, respx.Route], escalate: bool, +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + router: Final = make_router(settings={ + "enable_context_window_escalation": escalate, + "tiers": {"SIMPLE": "profile", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 14)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + output: Final = base_window // 4 + trigger: Final = int(base_window * ContextCompactionConfig().trigger_ratio) + assert trigger - output < router._count_pre_call_check_tokens(payload["messages"], None, payload) < trigger + response: Final = await router.acompletion(model="auto", max_tokens=output, **payload) + assert all(route.call_count == 0 for route in wire) + assert response.choices[0].message.content == "profile answer" + + +async def test_an_armed_compactor_does_not_keep_an_oversized_history_on_an_inference_profile_tier( + wire: tuple[respx.Route, respx.Route], +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + router: Final = make_router(settings={ + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": "profile", "MEDIUM": "wide", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + router.add_deployment(Deployment( + model_name="wide", litellm_params={ + "model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", + }, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64}, + )) + compactor, answer = wire + answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer")) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 7)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload) + assert base_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64 + response: Final = await invoke(router, "chat", payload) + assert compactor.call_count == 0 and answer.call_count == 1 + assert "MAPLE-47" in str(response) + + +@pytest.mark.parametrize("chosen", ["SIMPLE", "MEDIUM"]) +async def test_an_oversized_history_compacts_on_the_cheapest_compactable_tier_beside_an_inference_profile( + wire: tuple[respx.Route, respx.Route], chosen: str, +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + small_window: Final = base_window + base_window // 2 + router: Final = make_router(small_window, settings={ + "enable_context_window_escalation": True, + "keyword_tier_rules": [{"keywords": ["answer"], "tier": chosen}], + "tiers": {"SIMPLE": "profile", "MEDIUM": "small", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 5)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload) + assert small_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64 + response: Final = await invoke(router, "chat", payload) + compactor, answer = wire + assert compactor.call_count == 1 and answer.call_count == 1 + assert "Background detail" not in answer.calls[0].request.content.decode() + assert "MAPLE-47" in str(response) + + @pytest.mark.parametrize( ("surface", "failure"), [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")] diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index ef7b9e485d4..762508c4daf 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -14786,6 +14786,24 @@ _CJK_TURNS = [ ] +_BEDROCK_PROFILE_ARN: Final = ( + "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz" +) +_LLAMA_BASE_MODEL: Final = "meta.llama3-8b-instruct-v1:0" + + +def _turns_past(window: int) -> list[dict[str, str]]: + return [ + { + "role": "user", + "content": "Here is everything discussed so far. " + + "The meeting notes were saved to the shared folder for later review this week. " * (window // 4), + }, + {"role": "assistant", "content": "Noted, I have read all of it."}, + {"role": "user", "content": "ok continue"}, + ] + + def _tier_config(**overrides: object) -> dict[str, object]: return { "tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, @@ -14825,6 +14843,127 @@ class TestContextWindowEscalation: assert result.routing_decision["tier"] == "COMPLEX" assert "context_escalation" in result.routing_decision["signals"] + @staticmethod + def _base_model_tier_router( + small: dict[str, object], base_window: int, **overrides: object + ) -> ComplexityRouter: + litellm_router: Final = Router( + model_list=[ + {"model_name": "small-model", **small}, + { + "model_name": "big-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": base_window * 16}, + }, + ] + ) + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=litellm_router, + complexity_router_config=_tier_config(**overrides), + ) + + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize( + "overrides", + [{}, {"adaptive": True, "session_affinity": True, "deployment_affinity": True}], + ids=["fixed", "adaptive"], + ) + @pytest.mark.parametrize("declared_in", ["model_info", "litellm_params"]) + async def test_an_inference_profile_window_declared_only_through_base_model_escalates_too( + self, overrides: dict[str, object], declared_in: str + ): + base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] + assert base_window is not None + router: Final = self._base_model_tier_router( + { + "litellm_params": { + "model": _BEDROCK_PROFILE_ARN, + "mock_response": "ok", + **({"base_model": _LLAMA_BASE_MODEL} if declared_in == "litellm_params" else {}), + }, + "model_info": {"base_model": _LLAMA_BASE_MODEL} if declared_in == "model_info" else {}, + }, + base_window, + **overrides, + ) + + result: Final = await router.async_pre_routing_hook( + model="test-router", + request_kwargs={"metadata": {"session_id": "long-session"}}, + messages=_turns_past(base_window), + ) + + assert result is not None + assert result.model == "big-model" + assert result.routing_decision["context_escalation_original_tier"] == "SIMPLE" + assert "context_escalation" in result.routing_decision["signals"] + + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize( + ("own_window", "model"), + [ + ("declared", _BEDROCK_PROFILE_ARN), + ("registered", _BEDROCK_PROFILE_ARN), + (None, "openai/a-model-the-cost-map-does-not-know"), + (None, "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:provisioned-model/abc123xyz"), + (None, "bedrock/llama/arn:aws:bedrock:us-west-2:000000000000:imported-model/abc123xyz"), + ], + ids=["declared", "registered", "not_a_bedrock_arn", "provisioned_model", "imported_model"], + ) + async def test_base_model_never_overrides_an_own_window_or_reaches_a_non_profile_route( + self, own_window: str | None, model: str, monkeypatch: pytest.MonkeyPatch + ): + base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] + assert base_window is not None + if own_window == "registered": + monkeypatch.setitem( + litellm.model_cost, + _BEDROCK_PROFILE_ARN, + {"max_input_tokens": base_window * 8, "litellm_provider": "bedrock", "mode": "chat"}, + ) + router: Final = self._base_model_tier_router( + { + "litellm_params": {"model": model, "mock_response": "ok"}, + "model_info": { + "base_model": _LLAMA_BASE_MODEL, + **({"max_input_tokens": base_window * 8} if own_window == "declared" else {}), + }, + }, + base_window, + ) + + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=_turns_past(base_window) + ) + + assert result is not None + assert result.model == "small-model" + assert "context_escalated" not in result.routing_decision + + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + async def test_an_unmapped_base_model_leaves_a_bedrock_arn_tier_in_place(self): + base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] + assert base_window is not None + router: Final = self._base_model_tier_router( + { + "litellm_params": {"model": _BEDROCK_PROFILE_ARN, "mock_response": "ok"}, + "model_info": {"base_model": "anthropic.claude-not-in-the-cost-map-v1:0"}, + }, + base_window, + ) + + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=_turns_past(base_window) + ) + + assert result is not None + assert result.model == "small-model" + assert "context_escalated" not in result.routing_decision + @pytest.mark.asyncio async def test_a_prompt_that_fits_routes_exactly_as_before(self): """The gate must be invisible for normal traffic: same model, no escalation facts."""