diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 32dd64cc39e..7380569042e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1360,11 +1360,6 @@ def _inference_profile_base_model_window(deployment: Mapping[str, object]) -> in return None -def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool: - window, has_unknown = facts - return window is not None and not has_unknown and needed <= int(window * buffer) - - class _RequestContextFit(NamedTuple): facts: Mapping[str, tuple[int | None, bool]] needed: int | None @@ -1378,6 +1373,12 @@ class _RequestContextFit(NamedTuple): or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer) ) + def can_escalate_to(self, model: str, needed: int) -> bool: + window, has_unknown = self.facts.get(model, (None, True)) + if window is None or has_unknown: + return False + return model in self.compaction_owned or needed <= int(window * self.buffer) + class _ContextWindowPlacement(NamedTuple): """Where the context-window gate placed the request: the placement tier, the subset of its @@ -3681,7 +3682,6 @@ class ComplexityRouter(CustomLogger): fit: _RequestContextFit, needed: int, ) -> _ContextWindowPlacement | None: - buffer: Final = self.config.context_window_escalation_buffer in_tier: Final = tuple(group for group in pool if fit.accepts(group)) if in_tier and len(in_tier) == len(pool): return None @@ -3689,11 +3689,7 @@ class ComplexityRouter(CustomLogger): if in_tier: return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable) for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]: - proven = tuple( - group - for group in pools.get(name, ()) - if _group_provably_fits(fit.facts.get(group, (None, True)), needed, buffer) - ) + proven = tuple(group for group in pools.get(name, ()) if fit.can_escalate_to(group, needed)) if proven: return _ContextWindowPlacement( tier=name if self.config.has_custom_tiers else ComplexityTier(name), @@ -4071,7 +4067,7 @@ class ComplexityRouter(CustomLogger): if not context_recovery or candidate_tier == decided_tier or fit.needed is None - or _group_provably_fits(fit.facts.get(model, (None, True)), fit.needed, fit.buffer) + or fit.can_escalate_to(model, fit.needed) ) candidates: Final = tuple( peer diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 581c9943500..bb55c10edb8 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -30,9 +30,9 @@ - {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"} - {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"} - {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"} -- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3422", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} -- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1272", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} -- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1272", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} +- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3510", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} +- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} +- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix; runs only on a stack with the prompt_caching pre-call check enabled (E2E_PROMPT_CACHING_STACK)"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 8cc7bf0c2b4..f83459c6d76 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -989,12 +989,13 @@ class ProxyClient: # ---- LLM calls ------------------------------------------------------ @step("Send a /chat/completions request to {body.model}") - def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]: + def chat(self, key: str, body: ChatBody, timeout: float | None = None) -> Result[ChatResponse]: return self.transport.post( "/chat/completions", headers=self.transport.bearer(key), json=body, response_type=ChatResponse, + timeout=timeout, ) @step("Send a streaming /chat/completions request to {body.model}") diff --git a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py index f9cc33fcc68..c7c23edfc8b 100644 --- a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py +++ b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py @@ -47,6 +47,7 @@ UPPER_BACKEND: Final = "bedrock/converse/global.amazon.nova-2-lite-v1:0" MAX_TOKENS: Final = 8 OVERSIZED_PROMPT_CHARS: Final = 1_000_000 BEYOND_DECLARED_WINDOW_PROMPT_CHARS: Final = 400_000 +LONG_PROMPT_TIMEOUT_SECONDS: Final = 180.0 FILLER: Final = ( "On the morning of day {n}, the garden club met beside the old mill and talked about which flowers " "to grow along the river walk. Everyone brought warm bread and shared stories about the cold winter " @@ -175,7 +176,11 @@ class TestContextWindowEscalationFromBaseModel: ) response: Final = unwrap( - proxy.chat(key, _chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS))) + proxy.chat( + key, + _chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS)), + timeout=LONG_PROMPT_TIMEOUT_SECONDS, + ) ) _answered(response) @@ -198,7 +203,11 @@ class TestContextWindowEscalationFromBaseModel: ) response: Final = unwrap( - proxy.chat(key, _chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS))) + proxy.chat( + key, + _chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS)), + timeout=LONG_PROMPT_TIMEOUT_SECONDS, + ) ) _answered(response) diff --git a/tests/unit/router_strategy/complexity_router/test_context_compaction.py b/tests/unit/router_strategy/complexity_router/test_context_compaction.py index a5116551498..d970c709f72 100644 --- a/tests/unit/router_strategy/complexity_router/test_context_compaction.py +++ b/tests/unit/router_strategy/complexity_router/test_context_compaction.py @@ -377,6 +377,39 @@ async def test_an_armed_compactor_does_not_keep_an_oversized_history_on_an_infer assert "MAPLE-47" in str(response) +@pytest.mark.parametrize("chosen", ["SIMPLE", "MEDIUM"]) +async def test_an_oversized_history_compacts_on_the_cheapest_compactable_tier_beside_an_inference_profile( + wire: tuple[respx.Route, respx.Route], chosen: str, +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + small_window: Final = base_window + base_window // 2 + router: Final = make_router(small_window, settings={ + "enable_context_window_escalation": True, + "keyword_tier_rules": [{"keywords": ["answer"], "tier": chosen}], + "tiers": {"SIMPLE": "profile", "MEDIUM": "small", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 5)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload) + assert small_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64 + response: Final = await invoke(router, "chat", payload) + compactor, answer = wire + assert compactor.call_count == 1 and answer.call_count == 1 + assert "Background detail" not in answer.calls[0].request.content.decode() + assert "MAPLE-47" in str(response) + + @pytest.mark.parametrize( ("surface", "failure"), [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")]