From a10f41d662afbf5b45055b80c10f79e691c4d2ed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:54:25 -0700 Subject: [PATCH 1/5] fix(router): resolve a deployment's window from base_model for every provider --- litellm/router.py | 8 +- tests/e2e/coverage_registry/reliability.yaml | 3 + tests/e2e/models.py | 2 + ...ty_router_context_window_base_model_e2e.py | 219 ++++++++++++++++++ .../router_strategy/test_complexity_router.py | 46 ++++ tests/unit/test_router/test_router.py | 67 ++++++ 6 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py diff --git a/litellm/router.py b/litellm/router.py index 010e5946cb8..a5db681e0cc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11035,7 +11035,9 @@ class Router: Augment litellm info with additional params set in `model_info`. - For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set. + A deployment's `base_model` (model_info first, then litellm_params) is the cost-map key for every + provider, the way /model/info resolves it. Without one, azure needs the deployment name to be a + known azure key and every other provider resolves its own `model:`. Returns - ModelInfo - If found -> typed dict with max tokens, input cost, etc. @@ -11077,7 +11079,7 @@ class Router: ) ## SET MODEL TO 'model=' - if base_model is None + not azure - if custom_llm_provider == "azure" and base_model is None: + if base_model is None and custom_llm_provider == "azure": # Router init auto-registers every deployment name into # litellm.model_cost as a zeroed stub, so membership alone can't # tell a resolvable name apart; require usable limits/costs. @@ -11099,7 +11101,7 @@ class Router: "Could not identify azure model '%s'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models", _model, ) - elif custom_llm_provider != "azure": + elif base_model is None: model = _model if "*" in model: # only call pattern_router for wildcard models diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 7382cc215f2..896a4618cac 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -30,6 +30,9 @@ - {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"} - {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"} - {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"} +- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router.py:11081", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} +- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1238", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} +- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1238", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix; runs only on a stack with the prompt_caching pre-call check enabled (E2E_PROMPT_CACHING_STACK)"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index ccb5cd35280..d3dd4960761 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1319,6 +1319,8 @@ class ModelInfoBody(BaseModel): team_id: str | None = None allowed_fails: int | None = None allowed_fails_policy: dict[str, int] | None = None + base_model: str | None = None + max_input_tokens: int | None = None class ModelNewBody(BaseModel): diff --git a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py new file mode 100644 index 00000000000..863e9b9bcb1 --- /dev/null +++ b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py @@ -0,0 +1,219 @@ +"""Live e2e pins for the complexity router's context-window escalation when a tier's +window is declared only through ``model_info.base_model``. + +A Bedrock application inference profile ARN has no cost-map row of its own, so the +SIMPLE tier's window exists only through ``base_model``. ``Router.get_router_model_info`` +read base_model for azure deployments alone and resolved every other provider's +deployment by its own ``model:``, so the ARN came back with no window, the escalation +gate treated an unknown window as able to hold anything, and an oversized request the +classifier scored SIMPLE stayed on the 200k model and 400'd at Bedrock with "prompt is +too long" instead of moving to the 1M MEDIUM tier. + +The explicit-window control declares ``max_input_tokens`` below its model's own row and +sends a request that fits the row but not the declaration, so it escalates only because +a declared window is honored. It lives on its own backend model: the router registers a +deployment's ``model_info`` under the shared cost-map key of its ``model:``, so a declared +window on the ARN would leak to the base_model deployment and hide the bug. A short +request on the base_model router proves the gate only moves what does not fit. + +Every deployment is registered via /model/new (stage has no static config for these) +and the served deployment is read back from the spend log's ``model``, which stores +either the registered alias or the provider-prefixed form. The prompts are plain prose +with a "how many" ask, which keeps the heuristic classifier on SIMPLE, and the router +estimates their size as chars/4. +""" + +import os +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final + +import pytest + +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import ( + ChatBody, + ChatMessage, + ChatResponse, + KeyGenerateBody, + LiteLLMParamsBody, + ModelInfoBody, + ModelNewBody, + SpendLogRow, +) +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + +SIMPLE_BACKEND: Final = os.environ.get( + "E2E_BEDROCK_INFERENCE_PROFILE_ARN", + "bedrock/converse/arn:aws:bedrock:us-west-2:888602223428:application-inference-profile/sccejsk6r44v", +) +SIMPLE_BASE_MODEL: Final = "global.anthropic.claude-haiku-4-5-20251001-v1:0" +EXPLICIT_WINDOW_BACKEND: Final = "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" +DECLARED_WINDOW_TOKENS: Final = 60_000 +UPPER_BACKEND: Final = "bedrock/converse/global.amazon.nova-2-lite-v1:0" +MAX_TOKENS: Final = 8 +OVERSIZED_PROMPT_CHARS: Final = 1_000_000 +BEYOND_DECLARED_WINDOW_PROMPT_CHARS: Final = 400_000 +FILLER: Final = ( + "On the morning of day {n}, the garden club met beside the old mill and talked about which flowers " + "to grow along the river walk. Everyone brought warm bread and shared stories about the cold winter " + "and the first robin of spring.\n" +) +ASK: Final = "How many days are written above? Reply with the single word OK." + + +@dataclass(frozen=True, slots=True) +class EscalationRouters: + base_model_router: str + explicit_window_router: str + simple_from_base_model: str + simple_with_explicit_window: str + upper: str + + +def _bedrock_params(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=model, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="us-west-2", + ) + + +def _router_params(simple: str, upper: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "heuristic", + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": simple, "MEDIUM": upper, "COMPLEX": upper, "REASONING": upper}, + }, + ) + + +def _prompt_of(chars: int) -> str: + marker: Final = unique_marker() + line_count: Final = -(-chars // len(FILLER.format(n=f"1 ({marker})"))) + lines: Final = tuple(FILLER.format(n=f"{n} ({marker})") for n in range(1, line_count + 1)) + return "".join(lines) + "\n" + ASK + + +def _chat_body(model: str, content: str) -> ChatBody: + return ChatBody(model=model, messages=[ChatMessage(role="user", content=content)], max_tokens=MAX_TOKENS) + + +def _key_for(proxy: ProxyClient, resources: ResourceManager, models: list[str]) -> str: + key: Final = proxy.generate_key(KeyGenerateBody(models=models, user_id="e2e-context-window-base-model")) + resources.defer(lambda: proxy.delete_key(key)) + return key + + +def _served_by(rows: list[SpendLogRow], alias: str, backend: str, context: str) -> None: + served: Final = tuple(row.model for row in rows) + assert served and all(model in {alias, backend} for model in served), ( + f"{context}: expected every request to be served by {alias} ({backend}), spend logs show {served}" + ) + + +def _answered(response: ChatResponse) -> None: + message: Final = response.choices[0].message if response.choices else None + assert message is not None and message.content, f"expected an answer, got {response}" + + +@pytest.fixture(scope="class") +def router_stack() -> Iterator[ExitStack]: + with ExitStack() as stack: + yield stack + + +@pytest.fixture(scope="class") +def routers(proxy: ProxyClient, router_stack: ExitStack) -> EscalationRouters: + marker: Final = unique_marker() + named: Final = EscalationRouters( + base_model_router=f"e2e-cw-router-base-{marker}", + explicit_window_router=f"e2e-cw-router-explicit-{marker}", + simple_from_base_model=f"e2e-cw-simple-base-{marker}", + simple_with_explicit_window=f"e2e-cw-simple-explicit-{marker}", + upper=f"e2e-cw-upper-{marker}", + ) + registrations: Final = ( + ModelNewBody( + model_name=named.simple_from_base_model, + litellm_params=_bedrock_params(SIMPLE_BACKEND), + model_info=ModelInfoBody(base_model=SIMPLE_BASE_MODEL), + ), + ModelNewBody( + model_name=named.simple_with_explicit_window, + litellm_params=_bedrock_params(EXPLICIT_WINDOW_BACKEND), + model_info=ModelInfoBody(max_input_tokens=DECLARED_WINDOW_TOKENS), + ), + ModelNewBody(model_name=named.upper, litellm_params=_bedrock_params(UPPER_BACKEND), model_info=ModelInfoBody()), + ModelNewBody( + model_name=named.base_model_router, + litellm_params=_router_params(named.simple_from_base_model, named.upper), + model_info=ModelInfoBody(), + ), + ModelNewBody( + model_name=named.explicit_window_router, + litellm_params=_router_params(named.simple_with_explicit_window, named.upper), + model_info=ModelInfoBody(), + ), + ) + for body in registrations: + router_stack.callback(proxy.delete_model, proxy.register_model(body)) + return named + + +class TestContextWindowEscalationFromBaseModel: + @pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_base_model_window") + def test_base_model_window_escalates( + self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters + ) -> None: + key: Final = _key_for( + proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper] + ) + + response: Final = unwrap( + proxy.chat(key, _chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS))) + ) + + _answered(response) + _served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "oversized ask, window from base_model") + + @pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_explicit_window") + def test_explicit_window_escalates( + self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters + ) -> None: + key: Final = _key_for( + proxy, resources, [routers.explicit_window_router, routers.simple_with_explicit_window, routers.upper] + ) + + response: Final = unwrap( + proxy.chat(key, _chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS))) + ) + + _answered(response) + _served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "ask beyond the declared window") + + @pytest.mark.covers("reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier") + def test_fitting_prompt_stays_on_the_simple_tier( + self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters + ) -> None: + key: Final = _key_for( + proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper] + ) + + response: Final = unwrap(proxy.chat(key, _chat_body(routers.base_model_router, f"say hello {unique_marker()}"))) + + _answered(response) + _served_by( + proxy.poll_logs_for_key(key), + routers.simple_from_base_model, + SIMPLE_BACKEND, + "short ask, window from base_model", + ) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 67e0649958b..822720bf86b 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -14225,6 +14225,52 @@ class TestContextWindowEscalation: assert result.routing_decision["tier"] == "COMPLEX" assert "context_escalation" in result.routing_decision["signals"] + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + async def test_a_window_declared_only_through_base_model_escalates_too(self): + """A Bedrock application inference profile ARN has no cost-map row; its window exists + only through `model_info.base_model`. The gate used to resolve that deployment to no + window at all (base_model was read for azure only), treat it as infinite, and leave an + oversized SIMPLE prompt on it to 400 at the provider.""" + base_model = "meta.llama3-8b-instruct-v1:0" + base_window = litellm.get_model_info(model=f"bedrock/{base_model}")["max_input_tokens"] + assert base_window is not None + litellm_router = Router( + model_list=[ + { + "model_name": "small-model", + "litellm_params": { + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "ok", + }, + "model_info": {"base_model": base_model}, + }, + { + "model_name": "big-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": base_window * 8}, + }, + ] + ) + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=litellm_router, + complexity_router_config=_tier_config(), + ) + filler_repeats = (base_window * 4) // len(_CONTEXT_FILLER) + 2 + oversized_turns = [ + {"role": "user", "content": "Here is everything discussed so far. " + _CONTEXT_FILLER * filler_repeats}, + {"role": "assistant", "content": "Noted, I have read all of it."}, + {"role": "user", "content": "ok continue"}, + ] + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=oversized_turns) + + assert result is not None + assert result.model == "big-model" + assert result.routing_decision["context_escalation_original_tier"] == "SIMPLE" + assert "context_escalation" in result.routing_decision["signals"] + @pytest.mark.asyncio async def test_a_prompt_that_fits_routes_exactly_as_before(self): """The gate must be invisible for normal traffic: same model, no escalation facts.""" diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 290b4e62852..9f8906a7f4a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -13650,6 +13650,73 @@ class TestAzureBaseModelFallbackLogging: ) assert model_info["max_input_tokens"] == litellm.model_cost["azure/gpt-4o-mini"]["max_input_tokens"] + +INFERENCE_PROFILE_ARN = "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz" +INFERENCE_PROFILE_BASE_MODEL = "global.anthropic.claude-haiku-4-5-20251001-v1:0" + + +@pytest.mark.usefixtures("local_model_cost_map") +class TestBaseModelResolvesWindowForEveryProvider: + """A Bedrock application inference profile ARN is not in the cost map, so its window and + prices only exist through `base_model`. get_router_model_info used to read base_model for + azure alone and resolved every other provider's deployment by its own `model:`, so the + ARN came back unmapped and the complexity router's context-window escalation, which + treats an unknown window as infinite, never moved an oversized request off that tier.""" + + def _router(self, model_info: dict[str, object], litellm_params: dict[str, object] | None = None) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "haiku", + "litellm_params": {"model": INFERENCE_PROFILE_ARN, **(litellm_params or {})}, + "model_info": {"id": "haiku-profile", **model_info}, + } + ] + ) + + def test_model_info_base_model_resolves_a_non_azure_deployment(self): + router = self._router({"base_model": INFERENCE_PROFILE_BASE_MODEL}) + + model_info = router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile") + + base = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}") + assert model_info["max_input_tokens"] == base["max_input_tokens"] + assert model_info["input_cost_per_token"] == base["input_cost_per_token"] + + def test_litellm_params_base_model_resolves_a_non_azure_deployment(self): + router = self._router({}, {"base_model": INFERENCE_PROFILE_BASE_MODEL}) + + model_info = router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile") + + base = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}") + assert model_info["max_input_tokens"] == base["max_input_tokens"] + + def test_explicit_max_input_tokens_still_wins_over_base_model(self): + base_window = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}")["max_input_tokens"] + assert base_window is not None + router = self._router({"base_model": INFERENCE_PROFILE_BASE_MODEL, "max_input_tokens": base_window // 2}) + + model_info = router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile") + + assert model_info["max_input_tokens"] == base_window // 2 + + def test_a_deployment_without_base_model_still_resolves_its_own_model(self): + router = litellm.Router( + model_list=[ + { + "model_name": "nova", + "litellm_params": {"model": "bedrock/converse/global.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "nova-plain"}, + } + ] + ) + + model_info = router.get_router_model_info(deployment=None, received_model_name="nova", id="nova-plain") + + own = litellm.get_model_info(model="bedrock/converse/global.amazon.nova-2-lite-v1:0") + assert model_info["max_input_tokens"] == own["max_input_tokens"] + + def test_model_group_info_intersects_supported_reasoning_efforts(): router = litellm.Router( model_list=[ From 7414d11d43f3001fe621d2235143fd4194434f55 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:27:44 -0700 Subject: [PATCH 2/5] test(router): mark the base_model window tests' locals Final --- .../router_strategy/test_complexity_router.py | 16 +++++---- tests/unit/test_router/test_router.py | 36 +++++++++++-------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 822720bf86b..8882c6f58ad 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -14232,10 +14232,10 @@ class TestContextWindowEscalation: only through `model_info.base_model`. The gate used to resolve that deployment to no window at all (base_model was read for azure only), treat it as infinite, and leave an oversized SIMPLE prompt on it to 400 at the provider.""" - base_model = "meta.llama3-8b-instruct-v1:0" - base_window = litellm.get_model_info(model=f"bedrock/{base_model}")["max_input_tokens"] + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=f"bedrock/{base_model}")["max_input_tokens"] assert base_window is not None - litellm_router = Router( + litellm_router: Final = Router( model_list=[ { "model_name": "small-model", @@ -14252,19 +14252,21 @@ class TestContextWindowEscalation: }, ] ) - router = ComplexityRouter( + router: Final = ComplexityRouter( model_name="test-router", litellm_router_instance=litellm_router, complexity_router_config=_tier_config(), ) - filler_repeats = (base_window * 4) // len(_CONTEXT_FILLER) + 2 - oversized_turns = [ + filler_repeats: Final = (base_window * 4) // len(_CONTEXT_FILLER) + 2 + oversized_turns: Final = [ {"role": "user", "content": "Here is everything discussed so far. " + _CONTEXT_FILLER * filler_repeats}, {"role": "assistant", "content": "Noted, I have read all of it."}, {"role": "user", "content": "ok continue"}, ] - result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=oversized_turns) + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=oversized_turns + ) assert result is not None assert result.model == "big-model" diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9f8906a7f4a..50d956f9d20 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -13651,8 +13651,10 @@ class TestAzureBaseModelFallbackLogging: assert model_info["max_input_tokens"] == litellm.model_cost["azure/gpt-4o-mini"]["max_input_tokens"] -INFERENCE_PROFILE_ARN = "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz" -INFERENCE_PROFILE_BASE_MODEL = "global.anthropic.claude-haiku-4-5-20251001-v1:0" +INFERENCE_PROFILE_ARN: Final = ( + "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz" +) +INFERENCE_PROFILE_BASE_MODEL: Final = "global.anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.usefixtures("local_model_cost_map") @@ -13675,33 +13677,39 @@ class TestBaseModelResolvesWindowForEveryProvider: ) def test_model_info_base_model_resolves_a_non_azure_deployment(self): - router = self._router({"base_model": INFERENCE_PROFILE_BASE_MODEL}) + router: Final = self._router({"base_model": INFERENCE_PROFILE_BASE_MODEL}) - model_info = router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile") + model_info: Final = router.get_router_model_info( + deployment=None, received_model_name="haiku", id="haiku-profile" + ) - base = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}") + base: Final = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}") assert model_info["max_input_tokens"] == base["max_input_tokens"] assert model_info["input_cost_per_token"] == base["input_cost_per_token"] def test_litellm_params_base_model_resolves_a_non_azure_deployment(self): - router = self._router({}, {"base_model": INFERENCE_PROFILE_BASE_MODEL}) + router: Final = self._router({}, {"base_model": INFERENCE_PROFILE_BASE_MODEL}) - model_info = router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile") + model_info: Final = router.get_router_model_info( + deployment=None, received_model_name="haiku", id="haiku-profile" + ) - base = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}") + base: Final = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}") assert model_info["max_input_tokens"] == base["max_input_tokens"] def test_explicit_max_input_tokens_still_wins_over_base_model(self): - base_window = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}")["max_input_tokens"] + base_window: Final = litellm.get_model_info(model=f"bedrock/{INFERENCE_PROFILE_BASE_MODEL}")["max_input_tokens"] assert base_window is not None - router = self._router({"base_model": INFERENCE_PROFILE_BASE_MODEL, "max_input_tokens": base_window // 2}) + router: Final = self._router({"base_model": INFERENCE_PROFILE_BASE_MODEL, "max_input_tokens": base_window // 2}) - model_info = router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile") + model_info: Final = router.get_router_model_info( + deployment=None, received_model_name="haiku", id="haiku-profile" + ) assert model_info["max_input_tokens"] == base_window // 2 def test_a_deployment_without_base_model_still_resolves_its_own_model(self): - router = litellm.Router( + router: Final = litellm.Router( model_list=[ { "model_name": "nova", @@ -13711,9 +13719,9 @@ class TestBaseModelResolvesWindowForEveryProvider: ] ) - model_info = router.get_router_model_info(deployment=None, received_model_name="nova", id="nova-plain") + model_info: Final = router.get_router_model_info(deployment=None, received_model_name="nova", id="nova-plain") - own = litellm.get_model_info(model="bedrock/converse/global.amazon.nova-2-lite-v1:0") + own: Final = litellm.get_model_info(model="bedrock/converse/global.amazon.nova-2-lite-v1:0") assert model_info["max_input_tokens"] == own["max_input_tokens"] From 73a999d5bea4fb2907c60be02f603dc194a22e7d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:52:05 -0700 Subject: [PATCH 3/5] fix(router): read a Bedrock ARN's base_model window only in context-window escalation A Bedrock ARN tier with no window of its own now escalates on its base_model's exact cost-map window. An own window, declared or registered, still wins, and compaction, pre-call checks and get_router_model_info keep main's behavior. Covers application inference profile, provisioned and imported model ARNs, and adds e2e metadata to the new e2e tests. --- litellm/llms/bedrock/common_utils.py | 4 + litellm/router.py | 22 +-- .../complexity_router/complexity_router.py | 25 +++- .../router_code_coverage.py | 4 +- ...ty_router_context_window_base_model_e2e.py | 26 +++- .../test_context_compaction.py | 30 ++++ .../router_strategy/test_complexity_router.py | 140 ++++++++++++++---- tests/unit/test_router/test_router.py | 49 ------ 8 files changed, 206 insertions(+), 94 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 0cf3e895ad2..5d76188d9df 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -981,6 +981,10 @@ def extract_model_name_from_bedrock_arn(model: str) -> str: return model +def is_bedrock_arn(model: str) -> bool: + return re.search(r"\barn:aws[\w-]*:bedrock:", model) is not None + + def is_bedrock_application_inference_profile_arn(model: str) -> bool: """ An application inference profile ARN ends in an opaque id with no provider diff --git a/litellm/router.py b/litellm/router.py index a507518a748..799c208a241 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -135,7 +135,7 @@ from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, vector_store_request_metadata, ) -from litellm.llms.bedrock.common_utils import get_bedrock_base_model, is_bedrock_application_inference_profile_arn +from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.model_info import ( @@ -1104,7 +1104,7 @@ def as_output_cap(value: object) -> int | None: return cap if cap >= 0 else None -def _exact_base_model_max_input_tokens(base_model: str) -> int | None: +def exact_base_model_max_input_tokens(base_model: str) -> int | None: try: model_info: Final = litellm.get_model_info(model=base_model) except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model @@ -1133,22 +1133,15 @@ def _declared_base_model(user_model_info: Mapping[str, object]) -> str | None: return None -def _base_model_window( +def _base_model_widened_window( route_model_info: ModelMapInfo, base_model: str | None, custom_llm_provider: str ) -> Mapping[str, int]: - if base_model is None: - return MappingProxyType({}) - if is_bedrock_application_inference_profile_arn(route_model_info["key"]): - profile_max_input_tokens: Final = _exact_base_model_max_input_tokens(base_model) - return MappingProxyType( - {} if profile_max_input_tokens is None else {"max_input_tokens": profile_max_input_tokens} - ) route_max_input_tokens: Final = route_model_info.get("max_input_tokens") - if route_max_input_tokens is None: + if base_model is None or route_max_input_tokens is None: return MappingProxyType({}) if not _names_route_model(base_model, route_model_info["key"], custom_llm_provider): return MappingProxyType({}) - base_max_input_tokens: Final = _exact_base_model_max_input_tokens(base_model) + base_max_input_tokens: Final = exact_base_model_max_input_tokens(base_model) if base_max_input_tokens is None or base_max_input_tokens <= route_max_input_tokens: return MappingProxyType({}) return MappingProxyType({"max_input_tokens": base_max_input_tokens}) @@ -11993,8 +11986,7 @@ class Router: Augment litellm info with additional params set in `model_info`. For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set. - For every other provider, a base_model naming the same model as `model:` can only widen its context window, - and a Bedrock application inference profile ARN, which names no model, takes its base_model's window. + For every other provider, a base_model naming the same model as `model:` can only widen its context window. Returns - ModelInfo - If found -> typed dict with max tokens, input cost, etc. @@ -12101,7 +12093,7 @@ class Router: # values are skipped or Deployment's None pricing defaults would erase the map's merged_model_info: Final[ModelMapInfo] = { **copy.deepcopy(model_info), - **_base_model_window( + **_base_model_widened_window( model_info, _declared_base_model(user_model_info) or _declared_base_model(litellm_params.model_extra or {}), custom_llm_provider, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..084130b336a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1273,6 +1273,28 @@ def _window_can_hold(window: int | None, needed: int, buffer: float) -> bool: return window is None or needed <= int(window * buffer) +def _declared_base_model(deployment: Mapping[str, object]) -> str | None: + match deployment: + case {"model_info": {"base_model": str() as base_model}} | { + "litellm_params": {"base_model": str() as base_model} + }: + return base_model + case _: + return None + + +def _bedrock_arn_base_model_window(deployment: Mapping[str, object]) -> int | None: + from litellm.llms.bedrock.common_utils import is_bedrock_arn + from litellm.router import exact_base_model_max_input_tokens + + base_model: Final = _declared_base_model(deployment) + match deployment: + case {"litellm_params": {"model": str() as model}} if base_model is not None and is_bedrock_arn(model): + return exact_base_model_max_input_tokens(base_model) + case _: + return None + + def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool: window, has_unknown = facts return window is not None and not has_unknown and needed <= int(window * buffer) @@ -3391,7 +3413,8 @@ class ComplexityRouter(CustomLogger): return name if self.config.has_custom_tiers else ComplexityTier(name) def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None: - return self._deployment_limit(group, deployment, "max_input_tokens") + window: Final = self._deployment_limit(group, deployment, "max_input_tokens") + return window if window is not None else _bedrock_arn_base_model_window(deployment) def _deployment_limit( self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"] diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 75cac8b026c..dc6fa602ddf 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -138,8 +138,8 @@ ignored_function_names = [ "_pre_call_check_requested_chat_params", # Tested through the unsupported-param pre-call check tests in test_router.py "_deployment_supports_param", # Tested through the deployment-capability pre-call check tests in test_router.py "_capability_model_and_provider", # Tested through the authenticating-provider pre-call check test in test_router.py - "_exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py - "_base_model_window", # Tested through get_router_model_info in test_router_model_cost_isolation.py + "exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py + "_base_model_widened_window", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_declared_base_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py diff --git a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py index 52c617645d6..f9cc33fcc68 100644 --- a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py +++ b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py @@ -18,9 +18,9 @@ from dataclasses import dataclass from typing import Final import pytest - from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import ( ChatBody, @@ -159,6 +159,14 @@ def routers(proxy: ProxyClient, router_stack: ExitStack) -> EscalationRouters: class TestContextWindowEscalationFromBaseModel: @pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_base_model_window") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.BEDROCK,), + models=(SIMPLE_BACKEND, UPPER_BACKEND), + mode=Mode.NONSTREAM, + ) + ) def test_base_model_window_escalates( self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters ) -> None: @@ -174,6 +182,14 @@ class TestContextWindowEscalationFromBaseModel: _served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "oversized ask, window from base_model") @pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_explicit_window") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.BEDROCK,), + models=(EXPLICIT_WINDOW_BACKEND, UPPER_BACKEND), + mode=Mode.NONSTREAM, + ) + ) def test_explicit_window_escalates( self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters ) -> None: @@ -189,6 +205,14 @@ class TestContextWindowEscalationFromBaseModel: _served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "ask beyond the declared window") @pytest.mark.covers("reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.BEDROCK,), + models=(SIMPLE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_fitting_prompt_stays_on_the_simple_tier( self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters ) -> None: diff --git a/tests/unit/router_strategy/complexity_router/test_context_compaction.py b/tests/unit/router_strategy/complexity_router/test_context_compaction.py index c6242f775c8..07e62bd0ff5 100644 --- a/tests/unit/router_strategy/complexity_router/test_context_compaction.py +++ b/tests/unit/router_strategy/complexity_router/test_context_compaction.py @@ -312,6 +312,36 @@ async def test_undersized_native_compactor_does_not_block_explicit_escalation( assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer" +@pytest.mark.parametrize("escalate", [False, True]) +async def test_a_bedrock_arn_tier_windowed_only_through_base_model_is_never_compacted( + wire: tuple[respx.Route, respx.Route], escalate: bool, +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + router: Final = make_router(settings={ + "enable_context_window_escalation": escalate, + "tiers": {"SIMPLE": "profile", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 14)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + output: Final = base_window // 4 + trigger: Final = int(base_window * ContextCompactionConfig().trigger_ratio) + assert trigger - output < router._count_pre_call_check_tokens(payload["messages"], None, payload) < trigger + response: Final = await router.acompletion(model="auto", max_tokens=output, **payload) + assert all(route.call_count == 0 for route in wire) + assert response.choices[0].message.content == "profile answer" + + @pytest.mark.parametrize( ("surface", "failure"), [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")] diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 1bf89401584..90b4d7994f8 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -14354,6 +14354,24 @@ _CJK_TURNS = [ ] +_BEDROCK_PROFILE_ARN: Final = ( + "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz" +) +_LLAMA_BASE_MODEL: Final = "meta.llama3-8b-instruct-v1:0" + + +def _turns_past(window: int) -> list[dict[str, str]]: + return [ + { + "role": "user", + "content": "Here is everything discussed so far. " + + "The meeting notes were saved to the shared folder for later review this week. " * (window // 4), + }, + {"role": "assistant", "content": "Noted, I have read all of it."}, + {"role": "user", "content": "ok continue"}, + ] + + def _tier_config(**overrides: object) -> dict[str, object]: return { "tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, @@ -14393,43 +14411,52 @@ class TestContextWindowEscalation: assert result.routing_decision["tier"] == "COMPLEX" assert "context_escalation" in result.routing_decision["signals"] - @pytest.mark.asyncio - @pytest.mark.usefixtures("local_model_cost_map") - async def test_an_inference_profile_window_declared_only_through_base_model_escalates_too(self): - base_model: Final = "meta.llama3-8b-instruct-v1:0" - base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] - assert base_window is not None + @staticmethod + def _base_model_tier_router(small: dict[str, object], base_window: int) -> ComplexityRouter: litellm_router: Final = Router( model_list=[ - { - "model_name": "small-model", - "litellm_params": { - "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", - "mock_response": "ok", - }, - "model_info": {"base_model": base_model}, - }, + {"model_name": "small-model", **small}, { "model_name": "big-model", "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"max_input_tokens": base_window * 8}, + "model_info": {"max_input_tokens": base_window * 16}, }, ] ) - router: Final = ComplexityRouter( - model_name="test-router", - litellm_router_instance=litellm_router, - complexity_router_config=_tier_config(), + return ComplexityRouter( + model_name="test-router", litellm_router_instance=litellm_router, complexity_router_config=_tier_config() + ) + + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize( + "arn", + [ + _BEDROCK_PROFILE_ARN, + "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:provisioned-model/abc123xyz", + "bedrock/llama/arn:aws:bedrock:us-west-2:000000000000:imported-model/abc123xyz", + ], + ) + @pytest.mark.parametrize("declared_in", ["model_info", "litellm_params"]) + async def test_a_bedrock_arn_window_declared_only_through_base_model_escalates_too( + self, arn: str, declared_in: str + ): + base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] + assert base_window is not None + router: Final = self._base_model_tier_router( + { + "litellm_params": { + "model": arn, + "mock_response": "ok", + **({"base_model": _LLAMA_BASE_MODEL} if declared_in == "litellm_params" else {}), + }, + "model_info": {"base_model": _LLAMA_BASE_MODEL} if declared_in == "model_info" else {}, + }, + base_window, ) - filler_repeats: Final = (base_window * 4) // len(_CONTEXT_FILLER) + 2 - oversized_turns: Final = [ - {"role": "user", "content": "Here is everything discussed so far. " + _CONTEXT_FILLER * filler_repeats}, - {"role": "assistant", "content": "Noted, I have read all of it."}, - {"role": "user", "content": "ok continue"}, - ] result: Final = await router.async_pre_routing_hook( - model="test-router", request_kwargs={}, messages=oversized_turns + model="test-router", request_kwargs={}, messages=_turns_past(base_window) ) assert result is not None @@ -14437,6 +14464,67 @@ class TestContextWindowEscalation: assert result.routing_decision["context_escalation_original_tier"] == "SIMPLE" assert "context_escalation" in result.routing_decision["signals"] + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + @pytest.mark.parametrize("own_window", ["declared", "registered", "not_a_bedrock_arn"]) + async def test_base_model_never_overrides_an_own_window_or_reaches_a_non_arn_route( + self, own_window: str, monkeypatch: pytest.MonkeyPatch + ): + base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] + assert base_window is not None + if own_window == "registered": + monkeypatch.setitem( + litellm.model_cost, + _BEDROCK_PROFILE_ARN, + {"max_input_tokens": base_window * 8, "litellm_provider": "bedrock", "mode": "chat"}, + ) + router: Final = self._base_model_tier_router( + { + "litellm_params": { + "model": ( + "openai/a-model-the-cost-map-does-not-know" + if own_window == "not_a_bedrock_arn" + else _BEDROCK_PROFILE_ARN + ), + "mock_response": "ok", + }, + "model_info": { + "base_model": _LLAMA_BASE_MODEL, + **({"max_input_tokens": base_window * 8} if own_window == "declared" else {}), + }, + }, + base_window, + ) + + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=_turns_past(base_window) + ) + + assert result is not None + assert result.model == "small-model" + assert "context_escalated" not in result.routing_decision + + @pytest.mark.asyncio + @pytest.mark.usefixtures("local_model_cost_map") + async def test_an_unmapped_base_model_leaves_a_bedrock_arn_tier_in_place(self): + base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] + assert base_window is not None + router: Final = self._base_model_tier_router( + { + "litellm_params": {"model": _BEDROCK_PROFILE_ARN, "mock_response": "ok"}, + "model_info": {"base_model": "anthropic.claude-not-in-the-cost-map-v1:0"}, + }, + base_window, + ) + + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=_turns_past(base_window) + ) + + assert result is not None + assert result.model == "small-model" + assert "context_escalated" not in result.routing_decision + @pytest.mark.asyncio async def test_a_prompt_that_fits_routes_exactly_as_before(self): """The gate must be invisible for normal traffic: same model, no escalation facts.""" diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b0e777b7ad2..13ac55fa937 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -16263,55 +16263,6 @@ class TestAzureBaseModelFallbackLogging: ) assert model_info["max_input_tokens"] == litellm.model_cost["azure/gpt-4o-mini"]["max_input_tokens"] - -INFERENCE_PROFILE_ARN: Final = ( - "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz" -) -INFERENCE_PROFILE_BASE_MODEL: Final = "global.anthropic.claude-haiku-4-5-20251001-v1:0" - - -@pytest.mark.usefixtures("local_model_cost_map") -class TestApplicationInferenceProfileWindowFromBaseModel: - def _profile_window(self, model_info: dict[str, object], litellm_params: dict[str, object] | None = None) -> object: - router: Final = litellm.Router( - model_list=[ - { - "model_name": "haiku", - "litellm_params": {"model": INFERENCE_PROFILE_ARN, **(litellm_params or {})}, - "model_info": {"id": "haiku-profile", **model_info}, - } - ] - ) - return router.get_router_model_info(deployment=None, received_model_name="haiku", id="haiku-profile")[ - "max_input_tokens" - ] - - def test_model_info_base_model_sets_the_profile_window(self): - base_window: Final = litellm.get_model_info(model=INFERENCE_PROFILE_BASE_MODEL)["max_input_tokens"] - assert base_window is not None - - assert self._profile_window({"base_model": INFERENCE_PROFILE_BASE_MODEL}) == base_window - - def test_litellm_params_base_model_sets_the_profile_window(self): - base_window: Final = litellm.get_model_info(model=INFERENCE_PROFILE_BASE_MODEL)["max_input_tokens"] - assert base_window is not None - - assert self._profile_window({}, {"base_model": INFERENCE_PROFILE_BASE_MODEL}) == base_window - - def test_explicit_max_input_tokens_still_wins_over_base_model(self): - base_window: Final = litellm.get_model_info(model=INFERENCE_PROFILE_BASE_MODEL)["max_input_tokens"] - assert base_window is not None - - declared_window: Final = self._profile_window( - {"base_model": INFERENCE_PROFILE_BASE_MODEL, "max_input_tokens": base_window // 2} - ) - - assert declared_window == base_window // 2 - - def test_an_unmapped_base_model_leaves_the_profile_window_unknown(self): - assert self._profile_window({"base_model": "anthropic.claude-not-in-the-cost-map-v1:0"}) is None - - def test_model_group_info_intersects_supported_reasoning_efforts(): router = litellm.Router( model_list=[ From b556088b7e0c9fbe2bd58892ddfc97ce0f6912c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:03:18 -0700 Subject: [PATCH 4/5] fix(router): limit the base_model window to inference profiles and keep escalation where compaction cannot read the window --- litellm/llms/bedrock/common_utils.py | 4 -- .../complexity_router/complexity_router.py | 63 ++++++++++--------- tests/e2e/coverage_registry/reliability.yaml | 2 +- .../test_context_compaction.py | 35 +++++++++++ .../router_strategy/test_complexity_router.py | 53 +++++++++------- 5 files changed, 101 insertions(+), 56 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 5d76188d9df..0cf3e895ad2 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -981,10 +981,6 @@ def extract_model_name_from_bedrock_arn(model: str) -> str: return model -def is_bedrock_arn(model: str) -> bool: - return re.search(r"\barn:aws[\w-]*:bedrock:", model) is not None - - def is_bedrock_application_inference_profile_arn(model: str) -> bool: """ An application inference profile ARN ends in an opaque id with no provider diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 084130b336a..cc8a42c8a44 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1283,13 +1283,15 @@ def _declared_base_model(deployment: Mapping[str, object]) -> str | None: return None -def _bedrock_arn_base_model_window(deployment: Mapping[str, object]) -> int | None: - from litellm.llms.bedrock.common_utils import is_bedrock_arn +def _inference_profile_base_model_window(deployment: Mapping[str, object]) -> int | None: + from litellm.llms.bedrock.common_utils import is_bedrock_application_inference_profile_arn from litellm.router import exact_base_model_max_input_tokens base_model: Final = _declared_base_model(deployment) match deployment: - case {"litellm_params": {"model": str() as model}} if base_model is not None and is_bedrock_arn(model): + case {"litellm_params": {"model": str() as model}} if ( + base_model is not None and is_bedrock_application_inference_profile_arn(model) + ): return exact_base_model_max_input_tokens(base_model) case _: return None @@ -1304,9 +1306,14 @@ class _RequestContextFit(NamedTuple): facts: Mapping[str, tuple[int | None, bool]] needed: int | None buffer: float + compaction_owned: frozenset[str] = frozenset() def accepts(self, model: str) -> bool: - return self.needed is None or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer) + return ( + self.needed is None + or model in self.compaction_owned + or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer) + ) class _ContextWindowPlacement(NamedTuple): @@ -3414,7 +3421,14 @@ class ComplexityRouter(CustomLogger): def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None: window: Final = self._deployment_limit(group, deployment, "max_input_tokens") - return window if window is not None else _bedrock_arn_base_model_window(deployment) + return window if window is not None else _inference_profile_base_model_window(deployment) + + def _window_hidden_from_compaction(self, group: str) -> bool: + return any( + _inference_profile_base_model_window(deployment) is not None + and self._deployment_limit(group, deployment, "max_input_tokens") is None + for deployment in self._group_deployments(group) + ) def _deployment_limit( self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"] @@ -3528,15 +3542,19 @@ class ComplexityRouter(CustomLogger): resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object], ) -> _RequestContextFit: - if ( - compaction_pending(request_kwargs) - or not self.config.enable_context_window_escalation - or not resolved_messages - ): + if not self.config.enable_context_window_escalation or not resolved_messages: return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset( (self.config.default_model,) if self.config.default_model else () ) + compacting: Final = compaction_pending(request_kwargs) + compaction_owned: Final = ( + frozenset(name for name in names if not self._window_hidden_from_compaction(name)) + if compacting + else frozenset[str]() + ) + if compacting and compaction_owned == names: + return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) facts: Final = MappingProxyType({name: self._group_window_facts(name) for name in names}) known: Final = tuple(window for window, _ in facts.values() if window is not None) buffer: Final = self.config.context_window_escalation_buffer @@ -3544,7 +3562,7 @@ class ComplexityRouter(CustomLogger): min(known) * buffer ) needed: Final = await self._counted_request_tokens(resolved_messages, request_kwargs) if needs_count else None - return _RequestContextFit(facts=facts, needed=needed, buffer=buffer) + return _RequestContextFit(facts=facts, needed=needed, buffer=buffer, compaction_owned=compaction_owned) async def _context_window_placement( self, @@ -3558,11 +3576,7 @@ class ComplexityRouter(CustomLogger): (the placement stands). Only a real tokenizer count ever moves a request, escalation lands only on groups whose every deployment declares a fitting window, and a group with no resolvable window is never moved on faith in either direction.""" - if ( - compaction_pending(request_kwargs) - or not self.config.enable_context_window_escalation - or not resolved_messages - ): + if not self.config.enable_context_window_escalation or not resolved_messages: return None pools: Final = self._tier_pools() pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ())) @@ -3571,7 +3585,7 @@ class ComplexityRouter(CustomLogger): fit: Final = context_fit or await self._request_context_fit(resolved_messages, request_kwargs) if fit.needed is None: return None - return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, facts=fit.facts, needed=fit.needed) + return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, fit=fit, needed=fit.needed) def _placement_for_tokens( self, @@ -3579,28 +3593,21 @@ class ComplexityRouter(CustomLogger): tier: ComplexityTier | str, pool: tuple[str, ...], pools: Mapping[str, list[str]], - facts: Mapping[str, tuple[int | None, bool]], + fit: _RequestContextFit, needed: int, ) -> _ContextWindowPlacement | None: buffer: Final = self.config.context_window_escalation_buffer - in_tier: Final = tuple( - group for group in pool if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer) - ) + in_tier: Final = tuple(group for group in pool if fit.accepts(group)) if in_tier and len(in_tier) == len(pool): return None - holdable: Final = frozenset( - group - for tier_pool in pools.values() - for group in tier_pool - if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer) - ) + holdable: Final = frozenset(group for tier_pool in pools.values() for group in tier_pool if fit.accepts(group)) if in_tier: return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable) for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]: proven = tuple( group for group in pools.get(name, ()) - if _group_provably_fits(facts.get(group, (None, True)), needed, buffer) + if _group_provably_fits(fit.facts.get(group, (None, True)), needed, buffer) ) if proven: return _ContextWindowPlacement( diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 352f3f8bcca..581c9943500 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -30,7 +30,7 @@ - {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"} - {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"} - {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"} -- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router.py:1141", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} +- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3422", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} - {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1272", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} - {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1272", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} diff --git a/tests/unit/router_strategy/complexity_router/test_context_compaction.py b/tests/unit/router_strategy/complexity_router/test_context_compaction.py index 07e62bd0ff5..a5116551498 100644 --- a/tests/unit/router_strategy/complexity_router/test_context_compaction.py +++ b/tests/unit/router_strategy/complexity_router/test_context_compaction.py @@ -342,6 +342,41 @@ async def test_a_bedrock_arn_tier_windowed_only_through_base_model_is_never_comp assert response.choices[0].message.content == "profile answer" +async def test_an_armed_compactor_does_not_keep_an_oversized_history_on_an_inference_profile_tier( + wire: tuple[respx.Route, respx.Route], +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + router: Final = make_router(settings={ + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": "profile", "MEDIUM": "wide", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + router.add_deployment(Deployment( + model_name="wide", litellm_params={ + "model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", + }, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64}, + )) + compactor, answer = wire + answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer")) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 7)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload) + assert base_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64 + response: Final = await invoke(router, "chat", payload) + assert compactor.call_count == 0 and answer.call_count == 1 + assert "MAPLE-47" in str(response) + + @pytest.mark.parametrize( ("surface", "failure"), [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")] diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 90b4d7994f8..b1ab444e08d 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -14412,7 +14412,9 @@ class TestContextWindowEscalation: assert "context_escalation" in result.routing_decision["signals"] @staticmethod - def _base_model_tier_router(small: dict[str, object], base_window: int) -> ComplexityRouter: + def _base_model_tier_router( + small: dict[str, object], base_window: int, **overrides: object + ) -> ComplexityRouter: litellm_router: Final = Router( model_list=[ {"model_name": "small-model", **small}, @@ -14424,39 +14426,41 @@ class TestContextWindowEscalation: ] ) return ComplexityRouter( - model_name="test-router", litellm_router_instance=litellm_router, complexity_router_config=_tier_config() + model_name="test-router", + litellm_router_instance=litellm_router, + complexity_router_config=_tier_config(**overrides), ) @pytest.mark.asyncio @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - "arn", - [ - _BEDROCK_PROFILE_ARN, - "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:provisioned-model/abc123xyz", - "bedrock/llama/arn:aws:bedrock:us-west-2:000000000000:imported-model/abc123xyz", - ], + "overrides", + [{}, {"adaptive": True, "session_affinity": True, "deployment_affinity": True}], + ids=["fixed", "adaptive"], ) @pytest.mark.parametrize("declared_in", ["model_info", "litellm_params"]) - async def test_a_bedrock_arn_window_declared_only_through_base_model_escalates_too( - self, arn: str, declared_in: str + async def test_an_inference_profile_window_declared_only_through_base_model_escalates_too( + self, overrides: dict[str, object], declared_in: str ): base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] assert base_window is not None router: Final = self._base_model_tier_router( { "litellm_params": { - "model": arn, + "model": _BEDROCK_PROFILE_ARN, "mock_response": "ok", **({"base_model": _LLAMA_BASE_MODEL} if declared_in == "litellm_params" else {}), }, "model_info": {"base_model": _LLAMA_BASE_MODEL} if declared_in == "model_info" else {}, }, base_window, + **overrides, ) result: Final = await router.async_pre_routing_hook( - model="test-router", request_kwargs={}, messages=_turns_past(base_window) + model="test-router", + request_kwargs={"metadata": {"session_id": "long-session"}}, + messages=_turns_past(base_window), ) assert result is not None @@ -14466,9 +14470,19 @@ class TestContextWindowEscalation: @pytest.mark.asyncio @pytest.mark.usefixtures("local_model_cost_map") - @pytest.mark.parametrize("own_window", ["declared", "registered", "not_a_bedrock_arn"]) - async def test_base_model_never_overrides_an_own_window_or_reaches_a_non_arn_route( - self, own_window: str, monkeypatch: pytest.MonkeyPatch + @pytest.mark.parametrize( + ("own_window", "model"), + [ + ("declared", _BEDROCK_PROFILE_ARN), + ("registered", _BEDROCK_PROFILE_ARN), + (None, "openai/a-model-the-cost-map-does-not-know"), + (None, "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:provisioned-model/abc123xyz"), + (None, "bedrock/llama/arn:aws:bedrock:us-west-2:000000000000:imported-model/abc123xyz"), + ], + ids=["declared", "registered", "not_a_bedrock_arn", "provisioned_model", "imported_model"], + ) + async def test_base_model_never_overrides_an_own_window_or_reaches_a_non_profile_route( + self, own_window: str | None, model: str, monkeypatch: pytest.MonkeyPatch ): base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"] assert base_window is not None @@ -14480,14 +14494,7 @@ class TestContextWindowEscalation: ) router: Final = self._base_model_tier_router( { - "litellm_params": { - "model": ( - "openai/a-model-the-cost-map-does-not-know" - if own_window == "not_a_bedrock_arn" - else _BEDROCK_PROFILE_ARN - ), - "mock_response": "ok", - }, + "litellm_params": {"model": model, "mock_response": "ok"}, "model_info": { "base_model": _LLAMA_BASE_MODEL, **({"max_input_tokens": base_window * 8} if own_window == "declared" else {}), From 9e0d0c98aed5eec029cee832deba4ce463874782 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:21:41 -0700 Subject: [PATCH 5/5] fix(router): let escalation pick a compaction tier with a known window beside an inference profile With compaction armed and an inference profile tier windowed only through base_model, escalation now accepts a compaction-owned destination whose window is known, so an oversized history compacts on the cheapest compactable tier instead of skipping it. The long-prompt e2e calls get a 180 second client timeout and the coverage registry points at the moved lines. --- .../complexity_router/complexity_router.py | 20 +++++------ tests/e2e/coverage_registry/reliability.yaml | 6 ++-- tests/e2e/proxy_client.py | 3 +- ...ty_router_context_window_base_model_e2e.py | 13 ++++++-- .../test_context_compaction.py | 33 +++++++++++++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 32dd64cc39e..7380569042e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1360,11 +1360,6 @@ def _inference_profile_base_model_window(deployment: Mapping[str, object]) -> in return None -def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool: - window, has_unknown = facts - return window is not None and not has_unknown and needed <= int(window * buffer) - - class _RequestContextFit(NamedTuple): facts: Mapping[str, tuple[int | None, bool]] needed: int | None @@ -1378,6 +1373,12 @@ class _RequestContextFit(NamedTuple): or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer) ) + def can_escalate_to(self, model: str, needed: int) -> bool: + window, has_unknown = self.facts.get(model, (None, True)) + if window is None or has_unknown: + return False + return model in self.compaction_owned or needed <= int(window * self.buffer) + class _ContextWindowPlacement(NamedTuple): """Where the context-window gate placed the request: the placement tier, the subset of its @@ -3681,7 +3682,6 @@ class ComplexityRouter(CustomLogger): fit: _RequestContextFit, needed: int, ) -> _ContextWindowPlacement | None: - buffer: Final = self.config.context_window_escalation_buffer in_tier: Final = tuple(group for group in pool if fit.accepts(group)) if in_tier and len(in_tier) == len(pool): return None @@ -3689,11 +3689,7 @@ class ComplexityRouter(CustomLogger): if in_tier: return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable) for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]: - proven = tuple( - group - for group in pools.get(name, ()) - if _group_provably_fits(fit.facts.get(group, (None, True)), needed, buffer) - ) + proven = tuple(group for group in pools.get(name, ()) if fit.can_escalate_to(group, needed)) if proven: return _ContextWindowPlacement( tier=name if self.config.has_custom_tiers else ComplexityTier(name), @@ -4071,7 +4067,7 @@ class ComplexityRouter(CustomLogger): if not context_recovery or candidate_tier == decided_tier or fit.needed is None - or _group_provably_fits(fit.facts.get(model, (None, True)), fit.needed, fit.buffer) + or fit.can_escalate_to(model, fit.needed) ) candidates: Final = tuple( peer diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 581c9943500..bb55c10edb8 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -30,9 +30,9 @@ - {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"} - {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"} - {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"} -- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3422", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} -- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1272", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} -- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1272", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} +- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3510", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"} +- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"} +- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"} - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix; runs only on a stack with the prompt_caching pre-call check enabled (E2E_PROMPT_CACHING_STACK)"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 8cc7bf0c2b4..f83459c6d76 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -989,12 +989,13 @@ class ProxyClient: # ---- LLM calls ------------------------------------------------------ @step("Send a /chat/completions request to {body.model}") - def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]: + def chat(self, key: str, body: ChatBody, timeout: float | None = None) -> Result[ChatResponse]: return self.transport.post( "/chat/completions", headers=self.transport.bearer(key), json=body, response_type=ChatResponse, + timeout=timeout, ) @step("Send a streaming /chat/completions request to {body.model}") diff --git a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py index f9cc33fcc68..c7c23edfc8b 100644 --- a/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py +++ b/tests/e2e/router/test_complexity_router_context_window_base_model_e2e.py @@ -47,6 +47,7 @@ UPPER_BACKEND: Final = "bedrock/converse/global.amazon.nova-2-lite-v1:0" MAX_TOKENS: Final = 8 OVERSIZED_PROMPT_CHARS: Final = 1_000_000 BEYOND_DECLARED_WINDOW_PROMPT_CHARS: Final = 400_000 +LONG_PROMPT_TIMEOUT_SECONDS: Final = 180.0 FILLER: Final = ( "On the morning of day {n}, the garden club met beside the old mill and talked about which flowers " "to grow along the river walk. Everyone brought warm bread and shared stories about the cold winter " @@ -175,7 +176,11 @@ class TestContextWindowEscalationFromBaseModel: ) response: Final = unwrap( - proxy.chat(key, _chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS))) + proxy.chat( + key, + _chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS)), + timeout=LONG_PROMPT_TIMEOUT_SECONDS, + ) ) _answered(response) @@ -198,7 +203,11 @@ class TestContextWindowEscalationFromBaseModel: ) response: Final = unwrap( - proxy.chat(key, _chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS))) + proxy.chat( + key, + _chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS)), + timeout=LONG_PROMPT_TIMEOUT_SECONDS, + ) ) _answered(response) diff --git a/tests/unit/router_strategy/complexity_router/test_context_compaction.py b/tests/unit/router_strategy/complexity_router/test_context_compaction.py index a5116551498..d970c709f72 100644 --- a/tests/unit/router_strategy/complexity_router/test_context_compaction.py +++ b/tests/unit/router_strategy/complexity_router/test_context_compaction.py @@ -377,6 +377,39 @@ async def test_an_armed_compactor_does_not_keep_an_oversized_history_on_an_infer assert "MAPLE-47" in str(response) +@pytest.mark.parametrize("chosen", ["SIMPLE", "MEDIUM"]) +async def test_an_oversized_history_compacts_on_the_cheapest_compactable_tier_beside_an_inference_profile( + wire: tuple[respx.Route, respx.Route], chosen: str, +) -> None: + base_model: Final = "meta.llama3-8b-instruct-v1:0" + base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"] + assert base_window is not None + small_window: Final = base_window + base_window // 2 + router: Final = make_router(small_window, settings={ + "enable_context_window_escalation": True, + "keyword_tier_rules": [{"keywords": ["answer"], "tier": chosen}], + "tiers": {"SIMPLE": "profile", "MEDIUM": "small", "COMPLEX": "large", "REASONING": "large"}, + }) + router.add_deployment(Deployment( + model_name="profile", litellm_params={ + "model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz", + "mock_response": "profile answer", + }, model_info={"id": "profile-answer", "base_model": base_model}, + )) + payload: Final = {"messages": [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 5)}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer with the project code"}, + ]} + counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload) + assert small_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64 + response: Final = await invoke(router, "chat", payload) + compactor, answer = wire + assert compactor.call_count == 1 and answer.call_count == 1 + assert "Background detail" not in answer.calls[0].request.content.decode() + assert "MAPLE-47" in str(response) + + @pytest.mark.parametrize( ("surface", "failure"), [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")]