mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 9e0d0c98ae into c8cef1a95b
This commit is contained in:
commit
53fd99d817
9 changed files with 547 additions and 38 deletions
|
|
@ -1257,7 +1257,7 @@ def stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str,
|
|||
)
|
||||
|
||||
|
||||
def _exact_base_model_max_input_tokens(base_model: str) -> int | None:
|
||||
def exact_base_model_max_input_tokens(base_model: str) -> int | None:
|
||||
try:
|
||||
model_info: Final = litellm.get_model_info(model=base_model)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model
|
||||
|
|
@ -1294,7 +1294,7 @@ def _base_model_widened_window(
|
|||
return MappingProxyType({})
|
||||
if not _names_route_model(base_model, route_model_info["key"], custom_llm_provider):
|
||||
return MappingProxyType({})
|
||||
base_max_input_tokens: Final = _exact_base_model_max_input_tokens(base_model)
|
||||
base_max_input_tokens: Final = exact_base_model_max_input_tokens(base_model)
|
||||
if base_max_input_tokens is None or base_max_input_tokens <= route_max_input_tokens:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({"max_input_tokens": base_max_input_tokens})
|
||||
|
|
|
|||
|
|
@ -1342,18 +1342,48 @@ def _window_can_hold(window: int | None, needed: int, buffer: float) -> bool:
|
|||
return window is None or needed <= int(window * buffer)
|
||||
|
||||
|
||||
def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool:
|
||||
window, has_unknown = facts
|
||||
return window is not None and not has_unknown and needed <= int(window * buffer)
|
||||
def _declared_base_model(deployment: Mapping[str, object]) -> str | None:
|
||||
match deployment:
|
||||
case {"model_info": {"base_model": str() as base_model}} | {
|
||||
"litellm_params": {"base_model": str() as base_model}
|
||||
}:
|
||||
return base_model
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _inference_profile_base_model_window(deployment: Mapping[str, object]) -> int | None:
|
||||
from litellm.llms.bedrock.common_utils import is_bedrock_application_inference_profile_arn
|
||||
from litellm.router import exact_base_model_max_input_tokens
|
||||
|
||||
base_model: Final = _declared_base_model(deployment)
|
||||
match deployment:
|
||||
case {"litellm_params": {"model": str() as model}} if (
|
||||
base_model is not None and is_bedrock_application_inference_profile_arn(model)
|
||||
):
|
||||
return exact_base_model_max_input_tokens(base_model)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
class _RequestContextFit(NamedTuple):
|
||||
facts: Mapping[str, tuple[int | None, bool]]
|
||||
needed: int | None
|
||||
buffer: float
|
||||
compaction_owned: frozenset[str] = frozenset()
|
||||
|
||||
def accepts(self, model: str) -> bool:
|
||||
return self.needed is None or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer)
|
||||
return (
|
||||
self.needed is None
|
||||
or model in self.compaction_owned
|
||||
or _window_can_hold(self.facts.get(model, (None, True))[0], self.needed, self.buffer)
|
||||
)
|
||||
|
||||
def can_escalate_to(self, model: str, needed: int) -> bool:
|
||||
window, has_unknown = self.facts.get(model, (None, True))
|
||||
if window is None or has_unknown:
|
||||
return False
|
||||
return model in self.compaction_owned or needed <= int(window * self.buffer)
|
||||
|
||||
|
||||
class _ContextWindowPlacement(NamedTuple):
|
||||
|
|
@ -3502,7 +3532,15 @@ class ComplexityRouter(CustomLogger):
|
|||
return name if self.config.has_custom_tiers else ComplexityTier(name)
|
||||
|
||||
def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None:
|
||||
return self._deployment_limit(group, deployment, "max_input_tokens")
|
||||
window: Final = self._deployment_limit(group, deployment, "max_input_tokens")
|
||||
return window if window is not None else _inference_profile_base_model_window(deployment)
|
||||
|
||||
def _window_hidden_from_compaction(self, group: str) -> bool:
|
||||
return any(
|
||||
_inference_profile_base_model_window(deployment) is not None
|
||||
and self._deployment_limit(group, deployment, "max_input_tokens") is None
|
||||
for deployment in self._group_deployments(group)
|
||||
)
|
||||
|
||||
def _deployment_limit(
|
||||
self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"]
|
||||
|
|
@ -3615,15 +3653,19 @@ class ComplexityRouter(CustomLogger):
|
|||
resolved_messages: Sequence[Mapping[str, object]] | None,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> _RequestContextFit:
|
||||
if (
|
||||
compaction_pending(request_kwargs)
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
|
||||
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
|
||||
(self.config.default_model,) if self.config.default_model else ()
|
||||
)
|
||||
compacting: Final = compaction_pending(request_kwargs)
|
||||
compaction_owned: Final = (
|
||||
frozenset(name for name in names if not self._window_hidden_from_compaction(name))
|
||||
if compacting
|
||||
else frozenset[str]()
|
||||
)
|
||||
if compacting and compaction_owned == names:
|
||||
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
|
||||
facts: Final = MappingProxyType({name: self._group_window_facts(name) for name in names})
|
||||
known: Final = tuple(window for window, _ in facts.values() if window is not None)
|
||||
buffer: Final = self.config.context_window_escalation_buffer
|
||||
|
|
@ -3631,7 +3673,7 @@ class ComplexityRouter(CustomLogger):
|
|||
min(known) * buffer
|
||||
)
|
||||
needed: Final = await self._counted_request_tokens(resolved_messages, request_kwargs) if needs_count else None
|
||||
return _RequestContextFit(facts=facts, needed=needed, buffer=buffer)
|
||||
return _RequestContextFit(facts=facts, needed=needed, buffer=buffer, compaction_owned=compaction_owned)
|
||||
|
||||
async def _context_window_placement(
|
||||
self,
|
||||
|
|
@ -3645,11 +3687,7 @@ class ComplexityRouter(CustomLogger):
|
|||
(the placement stands). Only a real tokenizer count ever moves a request, escalation
|
||||
lands only on groups whose every deployment declares a fitting window, and a group
|
||||
with no resolvable window is never moved on faith in either direction."""
|
||||
if (
|
||||
compaction_pending(request_kwargs)
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
return None
|
||||
pools: Final = self._tier_pools()
|
||||
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))
|
||||
|
|
@ -3658,7 +3696,7 @@ class ComplexityRouter(CustomLogger):
|
|||
fit: Final = context_fit or await self._request_context_fit(resolved_messages, request_kwargs)
|
||||
if fit.needed is None:
|
||||
return None
|
||||
return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, facts=fit.facts, needed=fit.needed)
|
||||
return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, fit=fit, needed=fit.needed)
|
||||
|
||||
def _placement_for_tokens(
|
||||
self,
|
||||
|
|
@ -3666,29 +3704,17 @@ class ComplexityRouter(CustomLogger):
|
|||
tier: ComplexityTier | str,
|
||||
pool: tuple[str, ...],
|
||||
pools: Mapping[str, list[str]],
|
||||
facts: Mapping[str, tuple[int | None, bool]],
|
||||
fit: _RequestContextFit,
|
||||
needed: int,
|
||||
) -> _ContextWindowPlacement | None:
|
||||
buffer: Final = self.config.context_window_escalation_buffer
|
||||
in_tier: Final = tuple(
|
||||
group for group in pool if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer)
|
||||
)
|
||||
in_tier: Final = tuple(group for group in pool if fit.accepts(group))
|
||||
if in_tier and len(in_tier) == len(pool):
|
||||
return None
|
||||
holdable: Final = frozenset(
|
||||
group
|
||||
for tier_pool in pools.values()
|
||||
for group in tier_pool
|
||||
if _window_can_hold(facts.get(group, (None, True))[0], needed, buffer)
|
||||
)
|
||||
holdable: Final = frozenset(group for tier_pool in pools.values() for group in tier_pool if fit.accepts(group))
|
||||
if in_tier:
|
||||
return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable)
|
||||
for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]:
|
||||
proven = tuple(
|
||||
group
|
||||
for group in pools.get(name, ())
|
||||
if _group_provably_fits(facts.get(group, (None, True)), needed, buffer)
|
||||
)
|
||||
proven = tuple(group for group in pools.get(name, ()) if fit.can_escalate_to(group, needed))
|
||||
if proven:
|
||||
return _ContextWindowPlacement(
|
||||
tier=name if self.config.has_custom_tiers else ComplexityTier(name),
|
||||
|
|
@ -4067,7 +4093,7 @@ class ComplexityRouter(CustomLogger):
|
|||
if not context_recovery
|
||||
or candidate_tier == decided_tier
|
||||
or fit.needed is None
|
||||
or _group_provably_fits(fit.facts.get(model, (None, True)), fit.needed, fit.buffer)
|
||||
or fit.can_escalate_to(model, fit.needed)
|
||||
)
|
||||
candidates: Final = tuple(
|
||||
peer
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ ignored_function_names = [
|
|||
"_route_capability_pin", # Exercised through Decisions routing tests in test_router.py
|
||||
"_deployment_serves_route", # Exercised through mixed-group Decisions tests in test_router.py
|
||||
"_without_deployments_that_cannot_serve_route", # Exercised through Decisions routing tests in test_router.py
|
||||
"_exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"exact_base_model_max_input_tokens", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"_base_model_widened_window", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
|
|
|
|||
|
|
@ -30,6 +30,9 @@
|
|||
- {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"}
|
||||
- {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"}
|
||||
- {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"}
|
||||
- {id: reliability.routing.context_window_escalation.escalates_from_base_model_window, module: reliability, tier: P0, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_base_model_window], exercised_on: [chat_completions], source: "litellm/router_strategy/complexity_router/complexity_router.py:3510", fail_before_fix: proven, rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment's window comes only from model_info.base_model (a Bedrock application inference profile ARN), instead of staying put and 400ing at the provider"}
|
||||
- {id: reliability.routing.context_window_escalation.escalates_from_explicit_window, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [escalates_from_explicit_window], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "An oversized SIMPLE request moves to the next tier that fits when the SIMPLE deployment declares max_input_tokens explicitly"}
|
||||
- {id: reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier, module: reliability, tier: P1, behavior: routing, variant: context_window_escalation, assertions: [fitting_prompt_stays_on_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:1335", rationale: "A request that fits the SIMPLE deployment's base_model window is served by that deployment, so resolving the window never over-escalates"}
|
||||
- {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"}
|
||||
- {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix; runs only on a stack with the prompt_caching pre-call check enabled (E2E_PROMPT_CACHING_STACK)"}
|
||||
- {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"}
|
||||
|
|
|
|||
|
|
@ -1438,6 +1438,8 @@ class ModelInfoBody(BaseModel):
|
|||
allowed_fails: int | None = None
|
||||
allowed_fails_policy: dict[str, int] | None = None
|
||||
token_count_model: TokenCountModelBody | None = None
|
||||
base_model: str | None = None
|
||||
max_input_tokens: int | None = None
|
||||
|
||||
|
||||
class ModelNewBody(BaseModel):
|
||||
|
|
|
|||
|
|
@ -989,12 +989,13 @@ class ProxyClient:
|
|||
# ---- LLM calls ------------------------------------------------------
|
||||
|
||||
@step("Send a /chat/completions request to {body.model}")
|
||||
def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]:
|
||||
def chat(self, key: str, body: ChatBody, timeout: float | None = None) -> Result[ChatResponse]:
|
||||
return self.transport.post(
|
||||
"/chat/completions",
|
||||
headers=self.transport.bearer(key),
|
||||
json=body,
|
||||
response_type=ChatResponse,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
@step("Send a streaming /chat/completions request to {body.model}")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
"""Live e2e pins for the complexity router's context-window escalation when a tier is a Bedrock
|
||||
application inference profile ARN whose window is declared only through ``model_info.base_model``.
|
||||
|
||||
The ARN names no model, so before the router took the base_model's window it saw no window at all,
|
||||
the escalation gate treated that as able to hold anything, and an oversized request scored SIMPLE
|
||||
stayed on the 200k model and 400'd at Bedrock instead of moving to the 1M MEDIUM tier.
|
||||
|
||||
The explicit-window control declares ``max_input_tokens`` below its model's own row on a separate
|
||||
backend model, since the router registers a deployment's ``model_info`` under the shared cost-map key
|
||||
of its ``model:`` and a declared window on the ARN would leak to the base_model deployment. A short
|
||||
request on the base_model router proves the gate only moves what does not fit.
|
||||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import unwrap
|
||||
from e2e_metadata import Domain, Mode, Provider, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import (
|
||||
ChatBody,
|
||||
ChatMessage,
|
||||
ChatResponse,
|
||||
KeyGenerateBody,
|
||||
LiteLLMParamsBody,
|
||||
ModelInfoBody,
|
||||
ModelNewBody,
|
||||
SpendLogRow,
|
||||
)
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
SIMPLE_BACKEND: Final = os.environ.get(
|
||||
"E2E_BEDROCK_INFERENCE_PROFILE_ARN",
|
||||
"bedrock/converse/arn:aws:bedrock:us-west-2:888602223428:application-inference-profile/sccejsk6r44v",
|
||||
)
|
||||
SIMPLE_BASE_MODEL: Final = "global.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
EXPLICIT_WINDOW_BACKEND: Final = "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
DECLARED_WINDOW_TOKENS: Final = 60_000
|
||||
UPPER_BACKEND: Final = "bedrock/converse/global.amazon.nova-2-lite-v1:0"
|
||||
MAX_TOKENS: Final = 8
|
||||
OVERSIZED_PROMPT_CHARS: Final = 1_000_000
|
||||
BEYOND_DECLARED_WINDOW_PROMPT_CHARS: Final = 400_000
|
||||
LONG_PROMPT_TIMEOUT_SECONDS: Final = 180.0
|
||||
FILLER: Final = (
|
||||
"On the morning of day {n}, the garden club met beside the old mill and talked about which flowers "
|
||||
"to grow along the river walk. Everyone brought warm bread and shared stories about the cold winter "
|
||||
"and the first robin of spring.\n"
|
||||
)
|
||||
ASK: Final = "How many days are written above? Reply with the single word OK."
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EscalationRouters:
|
||||
base_model_router: str
|
||||
explicit_window_router: str
|
||||
simple_from_base_model: str
|
||||
simple_with_explicit_window: str
|
||||
upper: str
|
||||
|
||||
|
||||
def _bedrock_params(model: str) -> LiteLLMParamsBody:
|
||||
return LiteLLMParamsBody(
|
||||
model=model,
|
||||
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
||||
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
|
||||
def _router_params(simple: str, upper: str) -> LiteLLMParamsBody:
|
||||
return LiteLLMParamsBody(
|
||||
model="auto_router/complexity_router",
|
||||
complexity_router_config={
|
||||
"classifier_type": "heuristic",
|
||||
"enable_context_window_escalation": True,
|
||||
"tiers": {"SIMPLE": simple, "MEDIUM": upper, "COMPLEX": upper, "REASONING": upper},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _prompt_of(chars: int) -> str:
|
||||
marker: Final = unique_marker()
|
||||
line_count: Final = -(-chars // len(FILLER.format(n=f"1 ({marker})")))
|
||||
lines: Final = tuple(FILLER.format(n=f"{n} ({marker})") for n in range(1, line_count + 1))
|
||||
return "".join(lines) + "\n" + ASK
|
||||
|
||||
|
||||
def _chat_body(model: str, content: str) -> ChatBody:
|
||||
return ChatBody(model=model, messages=[ChatMessage(role="user", content=content)], max_tokens=MAX_TOKENS)
|
||||
|
||||
|
||||
def _key_for(proxy: ProxyClient, resources: ResourceManager, models: list[str]) -> str:
|
||||
key: Final = proxy.generate_key(KeyGenerateBody(models=models, user_id="e2e-context-window-base-model"))
|
||||
resources.defer(lambda: proxy.delete_key(key))
|
||||
return key
|
||||
|
||||
|
||||
def _served_by(rows: list[SpendLogRow], alias: str, backend: str, context: str) -> None:
|
||||
served: Final = tuple(row.model for row in rows)
|
||||
assert served and all(model in {alias, backend} for model in served), (
|
||||
f"{context}: expected every request to be served by {alias} ({backend}), spend logs show {served}"
|
||||
)
|
||||
|
||||
|
||||
def _answered(response: ChatResponse) -> None:
|
||||
message: Final = response.choices[0].message if response.choices else None
|
||||
assert message is not None and message.content, f"expected an answer, got {response}"
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def router_stack() -> Iterator[ExitStack]:
|
||||
with ExitStack() as stack:
|
||||
yield stack
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def routers(proxy: ProxyClient, router_stack: ExitStack) -> EscalationRouters:
|
||||
marker: Final = unique_marker()
|
||||
named: Final = EscalationRouters(
|
||||
base_model_router=f"e2e-cw-router-base-{marker}",
|
||||
explicit_window_router=f"e2e-cw-router-explicit-{marker}",
|
||||
simple_from_base_model=f"e2e-cw-simple-base-{marker}",
|
||||
simple_with_explicit_window=f"e2e-cw-simple-explicit-{marker}",
|
||||
upper=f"e2e-cw-upper-{marker}",
|
||||
)
|
||||
registrations: Final = (
|
||||
ModelNewBody(
|
||||
model_name=named.simple_from_base_model,
|
||||
litellm_params=_bedrock_params(SIMPLE_BACKEND),
|
||||
model_info=ModelInfoBody(base_model=SIMPLE_BASE_MODEL),
|
||||
),
|
||||
ModelNewBody(
|
||||
model_name=named.simple_with_explicit_window,
|
||||
litellm_params=_bedrock_params(EXPLICIT_WINDOW_BACKEND),
|
||||
model_info=ModelInfoBody(max_input_tokens=DECLARED_WINDOW_TOKENS),
|
||||
),
|
||||
ModelNewBody(model_name=named.upper, litellm_params=_bedrock_params(UPPER_BACKEND), model_info=ModelInfoBody()),
|
||||
ModelNewBody(
|
||||
model_name=named.base_model_router,
|
||||
litellm_params=_router_params(named.simple_from_base_model, named.upper),
|
||||
model_info=ModelInfoBody(),
|
||||
),
|
||||
ModelNewBody(
|
||||
model_name=named.explicit_window_router,
|
||||
litellm_params=_router_params(named.simple_with_explicit_window, named.upper),
|
||||
model_info=ModelInfoBody(),
|
||||
),
|
||||
)
|
||||
for body in registrations:
|
||||
router_stack.callback(proxy.delete_model, proxy.register_model(body))
|
||||
return named
|
||||
|
||||
|
||||
class TestContextWindowEscalationFromBaseModel:
|
||||
@pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_base_model_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.ROUTING,
|
||||
providers=(Provider.BEDROCK,),
|
||||
models=(SIMPLE_BACKEND, UPPER_BACKEND),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_base_model_window_escalates(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters
|
||||
) -> None:
|
||||
key: Final = _key_for(
|
||||
proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper]
|
||||
)
|
||||
|
||||
response: Final = unwrap(
|
||||
proxy.chat(
|
||||
key,
|
||||
_chat_body(routers.base_model_router, _prompt_of(OVERSIZED_PROMPT_CHARS)),
|
||||
timeout=LONG_PROMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
)
|
||||
|
||||
_answered(response)
|
||||
_served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "oversized ask, window from base_model")
|
||||
|
||||
@pytest.mark.covers("reliability.routing.context_window_escalation.escalates_from_explicit_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.ROUTING,
|
||||
providers=(Provider.BEDROCK,),
|
||||
models=(EXPLICIT_WINDOW_BACKEND, UPPER_BACKEND),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_explicit_window_escalates(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters
|
||||
) -> None:
|
||||
key: Final = _key_for(
|
||||
proxy, resources, [routers.explicit_window_router, routers.simple_with_explicit_window, routers.upper]
|
||||
)
|
||||
|
||||
response: Final = unwrap(
|
||||
proxy.chat(
|
||||
key,
|
||||
_chat_body(routers.explicit_window_router, _prompt_of(BEYOND_DECLARED_WINDOW_PROMPT_CHARS)),
|
||||
timeout=LONG_PROMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
)
|
||||
|
||||
_answered(response)
|
||||
_served_by(proxy.poll_logs_for_key(key), routers.upper, UPPER_BACKEND, "ask beyond the declared window")
|
||||
|
||||
@pytest.mark.covers("reliability.routing.context_window_escalation.fitting_prompt_stays_on_tier")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.ROUTING,
|
||||
providers=(Provider.BEDROCK,),
|
||||
models=(SIMPLE_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_fitting_prompt_stays_on_the_simple_tier(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, routers: EscalationRouters
|
||||
) -> None:
|
||||
key: Final = _key_for(
|
||||
proxy, resources, [routers.base_model_router, routers.simple_from_base_model, routers.upper]
|
||||
)
|
||||
|
||||
response: Final = unwrap(proxy.chat(key, _chat_body(routers.base_model_router, f"say hello {unique_marker()}")))
|
||||
|
||||
_answered(response)
|
||||
_served_by(
|
||||
proxy.poll_logs_for_key(key),
|
||||
routers.simple_from_base_model,
|
||||
SIMPLE_BACKEND,
|
||||
"short ask, window from base_model",
|
||||
)
|
||||
|
|
@ -313,6 +313,104 @@ async def test_undersized_native_compactor_does_not_block_explicit_escalation(
|
|||
assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("escalate", [False, True])
|
||||
async def test_a_bedrock_arn_tier_windowed_only_through_base_model_is_never_compacted(
|
||||
wire: tuple[respx.Route, respx.Route], escalate: bool,
|
||||
) -> None:
|
||||
base_model: Final = "meta.llama3-8b-instruct-v1:0"
|
||||
base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"]
|
||||
assert base_window is not None
|
||||
router: Final = make_router(settings={
|
||||
"enable_context_window_escalation": escalate,
|
||||
"tiers": {"SIMPLE": "profile", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"},
|
||||
})
|
||||
router.add_deployment(Deployment(
|
||||
model_name="profile", litellm_params={
|
||||
"model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz",
|
||||
"mock_response": "profile answer",
|
||||
}, model_info={"id": "profile-answer", "base_model": base_model},
|
||||
))
|
||||
payload: Final = {"messages": [
|
||||
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 14)},
|
||||
{"role": "assistant", "content": "Recorded"},
|
||||
{"role": "user", "content": "Answer with the project code"},
|
||||
]}
|
||||
output: Final = base_window // 4
|
||||
trigger: Final = int(base_window * ContextCompactionConfig().trigger_ratio)
|
||||
assert trigger - output < router._count_pre_call_check_tokens(payload["messages"], None, payload) < trigger
|
||||
response: Final = await router.acompletion(model="auto", max_tokens=output, **payload)
|
||||
assert all(route.call_count == 0 for route in wire)
|
||||
assert response.choices[0].message.content == "profile answer"
|
||||
|
||||
|
||||
async def test_an_armed_compactor_does_not_keep_an_oversized_history_on_an_inference_profile_tier(
|
||||
wire: tuple[respx.Route, respx.Route],
|
||||
) -> None:
|
||||
base_model: Final = "meta.llama3-8b-instruct-v1:0"
|
||||
base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"]
|
||||
assert base_window is not None
|
||||
router: Final = make_router(settings={
|
||||
"enable_context_window_escalation": True,
|
||||
"tiers": {"SIMPLE": "profile", "MEDIUM": "wide", "COMPLEX": "large", "REASONING": "large"},
|
||||
})
|
||||
router.add_deployment(Deployment(
|
||||
model_name="profile", litellm_params={
|
||||
"model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz",
|
||||
"mock_response": "profile answer",
|
||||
}, model_info={"id": "profile-answer", "base_model": base_model},
|
||||
))
|
||||
router.add_deployment(Deployment(
|
||||
model_name="wide", litellm_params={
|
||||
"model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test",
|
||||
}, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64},
|
||||
))
|
||||
compactor, answer = wire
|
||||
answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer"))
|
||||
payload: Final = {"messages": [
|
||||
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 7)},
|
||||
{"role": "assistant", "content": "Recorded"},
|
||||
{"role": "user", "content": "Answer with the project code"},
|
||||
]}
|
||||
counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload)
|
||||
assert base_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64
|
||||
response: Final = await invoke(router, "chat", payload)
|
||||
assert compactor.call_count == 0 and answer.call_count == 1
|
||||
assert "MAPLE-47" in str(response)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chosen", ["SIMPLE", "MEDIUM"])
|
||||
async def test_an_oversized_history_compacts_on_the_cheapest_compactable_tier_beside_an_inference_profile(
|
||||
wire: tuple[respx.Route, respx.Route], chosen: str,
|
||||
) -> None:
|
||||
base_model: Final = "meta.llama3-8b-instruct-v1:0"
|
||||
base_window: Final = litellm.get_model_info(model=base_model)["max_input_tokens"]
|
||||
assert base_window is not None
|
||||
small_window: Final = base_window + base_window // 2
|
||||
router: Final = make_router(small_window, settings={
|
||||
"enable_context_window_escalation": True,
|
||||
"keyword_tier_rules": [{"keywords": ["answer"], "tier": chosen}],
|
||||
"tiers": {"SIMPLE": "profile", "MEDIUM": "small", "COMPLEX": "large", "REASONING": "large"},
|
||||
})
|
||||
router.add_deployment(Deployment(
|
||||
model_name="profile", litellm_params={
|
||||
"model": "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz",
|
||||
"mock_response": "profile answer",
|
||||
}, model_info={"id": "profile-answer", "base_model": base_model},
|
||||
))
|
||||
payload: Final = {"messages": [
|
||||
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * (base_window // 5)},
|
||||
{"role": "assistant", "content": "Recorded"},
|
||||
{"role": "user", "content": "Answer with the project code"},
|
||||
]}
|
||||
counted: Final = router._count_pre_call_check_tokens(payload["messages"], None, payload)
|
||||
assert small_window < counted < 32000 * ContextCompactionConfig().trigger_ratio - 64
|
||||
response: Final = await invoke(router, "chat", payload)
|
||||
compactor, answer = wire
|
||||
assert compactor.call_count == 1 and answer.call_count == 1
|
||||
assert "Background detail" not in answer.calls[0].request.content.decode()
|
||||
assert "MAPLE-47" in str(response)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("surface", "failure"),
|
||||
[(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")]
|
||||
|
|
|
|||
|
|
@ -14786,6 +14786,24 @@ _CJK_TURNS = [
|
|||
]
|
||||
|
||||
|
||||
_BEDROCK_PROFILE_ARN: Final = (
|
||||
"bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:application-inference-profile/abc123xyz"
|
||||
)
|
||||
_LLAMA_BASE_MODEL: Final = "meta.llama3-8b-instruct-v1:0"
|
||||
|
||||
|
||||
def _turns_past(window: int) -> list[dict[str, str]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Here is everything discussed so far. "
|
||||
+ "The meeting notes were saved to the shared folder for later review this week. " * (window // 4),
|
||||
},
|
||||
{"role": "assistant", "content": "Noted, I have read all of it."},
|
||||
{"role": "user", "content": "ok continue"},
|
||||
]
|
||||
|
||||
|
||||
def _tier_config(**overrides: object) -> dict[str, object]:
|
||||
return {
|
||||
"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"},
|
||||
|
|
@ -14825,6 +14843,127 @@ class TestContextWindowEscalation:
|
|||
assert result.routing_decision["tier"] == "COMPLEX"
|
||||
assert "context_escalation" in result.routing_decision["signals"]
|
||||
|
||||
@staticmethod
|
||||
def _base_model_tier_router(
|
||||
small: dict[str, object], base_window: int, **overrides: object
|
||||
) -> ComplexityRouter:
|
||||
litellm_router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "small-model", **small},
|
||||
{
|
||||
"model_name": "big-model",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"},
|
||||
"model_info": {"max_input_tokens": base_window * 16},
|
||||
},
|
||||
]
|
||||
)
|
||||
return ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=litellm_router,
|
||||
complexity_router_config=_tier_config(**overrides),
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[{}, {"adaptive": True, "session_affinity": True, "deployment_affinity": True}],
|
||||
ids=["fixed", "adaptive"],
|
||||
)
|
||||
@pytest.mark.parametrize("declared_in", ["model_info", "litellm_params"])
|
||||
async def test_an_inference_profile_window_declared_only_through_base_model_escalates_too(
|
||||
self, overrides: dict[str, object], declared_in: str
|
||||
):
|
||||
base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"]
|
||||
assert base_window is not None
|
||||
router: Final = self._base_model_tier_router(
|
||||
{
|
||||
"litellm_params": {
|
||||
"model": _BEDROCK_PROFILE_ARN,
|
||||
"mock_response": "ok",
|
||||
**({"base_model": _LLAMA_BASE_MODEL} if declared_in == "litellm_params" else {}),
|
||||
},
|
||||
"model_info": {"base_model": _LLAMA_BASE_MODEL} if declared_in == "model_info" else {},
|
||||
},
|
||||
base_window,
|
||||
**overrides,
|
||||
)
|
||||
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="test-router",
|
||||
request_kwargs={"metadata": {"session_id": "long-session"}},
|
||||
messages=_turns_past(base_window),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "big-model"
|
||||
assert result.routing_decision["context_escalation_original_tier"] == "SIMPLE"
|
||||
assert "context_escalation" in result.routing_decision["signals"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("own_window", "model"),
|
||||
[
|
||||
("declared", _BEDROCK_PROFILE_ARN),
|
||||
("registered", _BEDROCK_PROFILE_ARN),
|
||||
(None, "openai/a-model-the-cost-map-does-not-know"),
|
||||
(None, "bedrock/converse/arn:aws:bedrock:us-west-2:000000000000:provisioned-model/abc123xyz"),
|
||||
(None, "bedrock/llama/arn:aws:bedrock:us-west-2:000000000000:imported-model/abc123xyz"),
|
||||
],
|
||||
ids=["declared", "registered", "not_a_bedrock_arn", "provisioned_model", "imported_model"],
|
||||
)
|
||||
async def test_base_model_never_overrides_an_own_window_or_reaches_a_non_profile_route(
|
||||
self, own_window: str | None, model: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"]
|
||||
assert base_window is not None
|
||||
if own_window == "registered":
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
_BEDROCK_PROFILE_ARN,
|
||||
{"max_input_tokens": base_window * 8, "litellm_provider": "bedrock", "mode": "chat"},
|
||||
)
|
||||
router: Final = self._base_model_tier_router(
|
||||
{
|
||||
"litellm_params": {"model": model, "mock_response": "ok"},
|
||||
"model_info": {
|
||||
"base_model": _LLAMA_BASE_MODEL,
|
||||
**({"max_input_tokens": base_window * 8} if own_window == "declared" else {}),
|
||||
},
|
||||
},
|
||||
base_window,
|
||||
)
|
||||
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="test-router", request_kwargs={}, messages=_turns_past(base_window)
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "small-model"
|
||||
assert "context_escalated" not in result.routing_decision
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
async def test_an_unmapped_base_model_leaves_a_bedrock_arn_tier_in_place(self):
|
||||
base_window: Final = litellm.get_model_info(model=_LLAMA_BASE_MODEL)["max_input_tokens"]
|
||||
assert base_window is not None
|
||||
router: Final = self._base_model_tier_router(
|
||||
{
|
||||
"litellm_params": {"model": _BEDROCK_PROFILE_ARN, "mock_response": "ok"},
|
||||
"model_info": {"base_model": "anthropic.claude-not-in-the-cost-map-v1:0"},
|
||||
},
|
||||
base_window,
|
||||
)
|
||||
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="test-router", request_kwargs={}, messages=_turns_past(base_window)
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "small-model"
|
||||
assert "context_escalated" not in result.routing_decision
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_prompt_that_fits_routes_exactly_as_before(self):
|
||||
"""The gate must be invisible for normal traffic: same model, no escalation facts."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue