feat(router): compact oversized auto-router requests with native providers

This commit is contained in:
Tin Chi Lo 2026-09-17 07:28:27 -04:00
parent 3c34b92594
commit 1276df0b03
29 changed files with 2945 additions and 68 deletions

View file

@ -7,6 +7,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",

View file

@ -0,0 +1,151 @@
from __future__ import annotations
from collections.abc import Mapping
from copy import deepcopy
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from pydantic import StrictStr, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
NativeProtocol: TypeAlias = Literal["responses", "messages"]
_Items: TypeAlias = tuple[Mapping[str, object], ...]
_ITEMS: Final = TypeAdapter(_Items)
_WIRE_ITEMS: Final = TypeAdapter(list[Mapping[str, object]])
_HEADERS: Final = TypeAdapter(Mapping[StrictStr, StrictStr])
_COMPACTION_BETA: Final = "compact-2026-09-04"
class _CompactionOperation(TypedDict):
type: ReadOnly[Literal["summarize"]]
@dataclass(frozen=True, slots=True)
class CompactionHistoryError:
message: str
kind: Literal["compaction_history_error"] = "compaction_history_error"
@dataclass(frozen=True, slots=True)
class NativeHistory:
field: str
prefix: _Items
tail: _Items
def _copy_items(items: object) -> _Items:
return tuple(MappingProxyType(item) for item in deepcopy(_ITEMS.validate_python(items)))
def _native_items(value: object) -> _Items | CompactionHistoryError:
try:
return _copy_items(value)
except ValidationError:
return CompactionHistoryError("Native history requires an array of objects")
def _is_user_request(item: Mapping[str, object], protocol: NativeProtocol) -> bool:
if item.get("role") != "user":
return False
content: Final = item.get("content")
if protocol == "responses" or isinstance(content, str):
return True
blocks: Final = _native_items(content)
return isinstance(blocks, tuple) and any(block.get("type") != "tool_result" for block in blocks)
def split_native_history(
payload: Mapping[str, object], protocol: NativeProtocol
) -> NativeHistory | CompactionHistoryError:
if payload.get("previous_response_id") is not None or payload.get("conversation") is not None:
return CompactionHistoryError("Server-side conversation history is unavailable for native compaction")
field: Final = "input" if protocol == "responses" else "messages"
items: Final = _native_items(payload.get(field))
if isinstance(items, CompactionHistoryError):
return items
start: Final = next(
(index for index in range(len(items) - 1, -1, -1) if _is_user_request(items[index], protocol)),
-1,
)
if start < 0:
return CompactionHistoryError("Native history has no actual user request")
if start == 0:
return CompactionHistoryError("Native history has no compactable prefix")
return NativeHistory(field=field, prefix=items[:start], tail=items[start:])
def compaction_headers(existing: object) -> Mapping[str, str]:
headers: Final = _HEADERS.validate_python(MappingProxyType({}) if existing is None else existing)
betas: Final = tuple(
beta.strip()
for name, value in headers.items()
if name.lower() == "anthropic-beta"
for beta in value.split(",")
if beta.strip()
)
preserved: Final = MappingProxyType(
{name: value for name, value in headers.items() if name.lower() != "anthropic-beta"}
)
return MappingProxyType(
{
**preserved,
"anthropic-beta": ",".join(dict.fromkeys((*betas, _COMPACTION_BETA))),
}
)
def native_compaction_payload(
history: NativeHistory, payload: Mapping[str, object], protocol: NativeProtocol
) -> Mapping[str, object]:
prefix: Final = deepcopy(_WIRE_ITEMS.validate_python(history.prefix))
parameters: Final = ("instructions",) if protocol == "responses" else ("system", "tools")
context: Final = MappingProxyType({key: deepcopy(payload[key]) for key in parameters if key in payload})
if protocol == "responses":
return MappingProxyType({"input": prefix, **context})
operation: Final[_CompactionOperation] = {"type": "summarize"}
return MappingProxyType(
{
"messages": prefix,
**context,
"compaction": operation,
"max_tokens": 4096,
"extra_headers": _HEADERS.validate_python(compaction_headers(payload.get("extra_headers"))),
}
)
def _has_text(item: Mapping[str, object], field: str) -> bool:
value: Final = item.get(field)
return isinstance(value, str) and bool(value.strip())
def native_compaction_result(
response: Mapping[str, object], history: NativeHistory, protocol: NativeProtocol
) -> _Items | CompactionHistoryError:
if protocol == "responses":
output: Final = _native_items(response.get("output"))
if isinstance(output, CompactionHistoryError):
return output
compacted: Final = tuple(item for item in output if item.get("type") == "compaction")
if not compacted or any(not _has_text(item, "encrypted_content") for item in compacted):
return CompactionHistoryError("Native Responses compaction requires encrypted compaction output")
canonical: Final = tuple(
MappingProxyType({key: value for key, value in item.items() if key != "created_by" or value is not None})
if item.get("type") == "compaction"
else item
for item in output
)
return _ITEMS.validate_python((*canonical, *_copy_items(history.tail)))
blocks: Final = _native_items(response.get("content"))
if response.get("stop_reason") != "compaction" or isinstance(blocks, CompactionHistoryError) or len(blocks) != 1:
return CompactionHistoryError("Native Messages compaction requires one signed compaction block")
block: Final = blocks[0]
if block.get("type") != "compaction" or not _has_text(block, "content") or not _has_text(block, "signature"):
return CompactionHistoryError("Native Messages compaction requires content and signature")
return _ITEMS.validate_python(
(
MappingProxyType({"role": "assistant", "content": _WIRE_ITEMS.validate_python(blocks)}),
*_copy_items(history.tail),
)
)

View file

@ -4,18 +4,30 @@ Anthropic CountTokens API handler.
Uses httpx for HTTP requests instead of the Anthropic SDK.
"""
from typing import Any, Final
from collections.abc import Mapping
from types import MappingProxyType
from typing import Annotated, Any, Final
import httpx
from pydantic import Field, StrictStr, TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.prompt_templates.compaction import compaction_headers
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
_NATIVE_COUNT_FIELDS: Final = frozenset({"messages", "system", "tools", "tool_choice", "thinking"})
_NATIVE_HEADERS: Final = TypeAdapter(dict[StrictStr, StrictStr])
_NATIVE_OBJECT: Final = TypeAdapter(dict[str, object])
_NATIVE_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
_NATIVE_COUNT: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field(strict=True, ge=0)])
class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
"""
@ -24,6 +36,76 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
Uses httpx for HTTP requests, following the same pattern as BedrockCountTokensHandler.
"""
async def count_native_tokens(
self,
model: str,
payload: Mapping[str, object],
api_key: str | None,
api_base: str | None,
headers: Mapping[str, str],
timeout: float,
) -> int:
handler: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
try:
body: Final = _NATIVE_OBJECT.validate_python(
MappingProxyType(
{
"model": model,
**MappingProxyType(
{
key: value
for key, value in payload.items()
if key in _NATIVE_COUNT_FIELDS and value is not None
}
),
}
)
)
config: Final = AnthropicMessagesConfig()
messages_url: Final = httpx.URL(
config.get_complete_url(
api_base=api_base or litellm.api_base,
api_key=api_key,
model=model,
optional_params=_NATIVE_OBJECT.validate_python(MappingProxyType({})),
litellm_params=_NATIVE_OBJECT.validate_python(MappingProxyType({})),
)
)
compact_headers: Final = compaction_headers(headers)
beta_headers: Final = _NATIVE_HEADERS.validate_python(
MappingProxyType(
{
**compact_headers,
"anthropic-beta": ",".join(
dict.fromkeys(
(*compact_headers["anthropic-beta"].split(","), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION)
)
),
}
)
)
anthropic_headers: Final = _NATIVE_HEADERS.validate_python(
config.validate_anthropic_messages_environment(
headers=beta_headers,
model=model,
messages=_NATIVE_MESSAGES.validate_python(payload.get("messages")),
optional_params=body,
litellm_params=_NATIVE_OBJECT.validate_python(MappingProxyType({})),
api_key=api_key or litellm.anthropic_key or litellm.api_key,
api_base=str(messages_url),
)[0]
)
response: Final = await handler.client.post(
str(messages_url.copy_with(path=f"{messages_url.path.rstrip('/')}/count_tokens")),
headers=anthropic_headers,
json=body,
timeout=timeout,
)
response.raise_for_status()
return _NATIVE_COUNT.validate_python(_NATIVE_OBJECT.validate_json(response.content).get("input_tokens"))
except (httpx.HTTPError, ValueError):
raise ValueError("Provider-native token counting failed") from None
async def handle_count_tokens_request(
self,
model: str,

View file

@ -69,6 +69,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
"compaction",
"output_format",
"inference_geo",
"speed",

View file

@ -5,9 +5,13 @@ Uses httpx for HTTP requests to OpenAI's /v1/responses/input_tokens endpoint.
"""
import json
from typing import Any, Final
from collections.abc import Mapping
from types import MappingProxyType
from typing import Annotated, Any, Final
import httpx
from openai import APIError, AsyncOpenAI
from pydantic import Field, StrictStr, TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -16,6 +20,16 @@ from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.count_tokens.transformation import (
OpenAICountTokensConfig,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
_NATIVE_COUNT_FIELDS: Final = frozenset(
{"input", "instructions", "tools", "tool_choice", "parallel_tool_calls", "reasoning", "text"}
)
_NATIVE_HEADERS: Final = TypeAdapter(dict[StrictStr, StrictStr])
_NATIVE_OBJECT: Final = TypeAdapter(dict[str, object])
_NATIVE_COUNT: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field(strict=True, ge=0)])
class OpenAICountTokensHandler(OpenAICountTokensConfig):
@ -23,6 +37,64 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
Handler for OpenAI Responses API token counting requests.
"""
async def count_native_tokens(
self,
model: str,
payload: Mapping[str, object],
api_key: str | None,
api_base: str | None,
headers: Mapping[str, str],
timeout: float,
) -> int:
handler: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI)
try:
body: Final = _NATIVE_OBJECT.validate_python(
MappingProxyType(
{
"model": model,
**MappingProxyType(
{
key: value
for key, value in payload.items()
if key in _NATIVE_COUNT_FIELDS and value is not None
}
),
}
)
)
config: Final = OpenAIResponsesAPIConfig()
openai_base: Final = config.get_complete_url(
api_base=api_base, litellm_params=_NATIVE_OBJECT.validate_python(MappingProxyType({}))
).removesuffix("/responses")
openai_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
environment_headers: Final = _NATIVE_HEADERS.validate_python(
config.validate_environment(
headers=_NATIVE_HEADERS.validate_python(headers),
model=model,
litellm_params=GenericLiteLLMParams(api_key=openai_key, api_base=openai_base),
)
)
client: Final = AsyncOpenAI(
api_key=openai_key or "",
base_url=openai_base,
http_client=handler.client,
max_retries=0,
)
sdk_header_names: Final = MappingProxyType({name.lower(): name for name in client.default_headers})
openai_headers: Final = MappingProxyType(
{
sdk_header_names.get(name.lower(), name): value
for source in (environment_headers, headers)
for name, value in source.items()
}
)
result: Final = await client.responses.input_tokens.count(
model=model, extra_body=body, extra_headers=openai_headers, timeout=timeout
)
return _NATIVE_COUNT.validate_python(result.input_tokens)
except (APIError, httpx.HTTPError, ValueError):
raise ValueError("Provider-native token counting failed") from None
async def handle_count_tokens_request(
self,
model: str,

View file

@ -898,11 +898,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
self,
model: str,
input: str | ResponseInputParam,
response_api_optional_request_params: dict,
response_api_optional_request_params: Mapping[str, object],
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
headers: Mapping[str, object],
) -> tuple[str, dict[str, object]]:
"""
Transform the compact response API request into a URL and data
@ -914,16 +914,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
compact_path: Final = parsed_url.path.rstrip("/") + "/compact"
url: Final = str(parsed_url.copy_with(path=compact_path))
replay_safe_input, sanitized_tools = self._prepared_input_and_tools(
replay_safe_input, _ = self._prepared_input_and_tools(
model=model,
input=input,
tools=response_api_optional_request_params.get("tools"),
tools=None,
litellm_params=litellm_params,
)
if sanitized_tools is not None:
response_api_optional_request_params["tools"] = sanitized_tools
data: Final = dict(
ResponsesAPIRequestParams(model=model, input=replay_safe_input, **response_api_optional_request_params)
compact_fields: Final = frozenset(
{"instructions", "previous_response_id", "prompt_cache_key", "prompt_cache_retention"}
)
data: Final = TypeAdapter(dict[str, object]).validate_python(
MappingProxyType(
{
"model": model,
"input": replay_safe_input,
**MappingProxyType(
{
key: value
for key, value in response_api_optional_request_params.items()
if key in compact_fields
}
),
}
)
)
return url, data

View file

@ -75,6 +75,11 @@ from litellm.proxy.common_utils.callback_utils import (
get_logging_caching_headers,
get_remaining_tokens_and_requests_from_request_data,
)
from litellm.proxy.common_utils.context_compaction import (
mark_summary_provider_completed,
proxy_summary_executor_scope,
summary_request_is_private,
)
from litellm.proxy.common_utils.http_parsing_utils import (
get_client_requested_model,
get_tags_from_request_body,
@ -2336,7 +2341,7 @@ class ProxyBaseLLMRequestProcessing:
def _debug_log_request_payload(self) -> None:
"""Log request payload at DEBUG level, truncating if too large."""
if not verbose_proxy_logger.isEnabledFor(logging.DEBUG):
if summary_request_is_private() or not verbose_proxy_logger.isEnabledFor(logging.DEBUG):
return
_payload_str: Final = json.dumps(self.data, default=str)
if len(_payload_str) > MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG:
@ -2517,20 +2522,24 @@ class ProxyBaseLLMRequestProcessing:
user_model=user_model,
user_api_key_dict=user_api_key_dict,
)
llm_call_task: Final = asyncio.create_task(llm_call)
tasks.append(llm_call_task)
with proxy_summary_executor_scope(
request, self.data, getattr(proxy_logging_obj, "max_parallel_request_limiter", None)
):
llm_call_task: Final = asyncio.create_task(llm_call)
tasks.append(llm_call_task)
llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call
llm_responses: Final = asyncio.gather(*tasks)
try:
if general_settings.get("cancel_on_disconnect", False):
responses = await _await_llm_call_cancelling_on_disconnect(request, llm_responses)
else:
responses = await llm_responses
finally:
await _cancel_pending_gather_tasks(tasks)
try:
if general_settings.get("cancel_on_disconnect", False):
responses = await _await_llm_call_cancelling_on_disconnect(request, llm_responses)
else:
responses = await llm_responses
finally:
await _cancel_pending_gather_tasks(tasks)
response = responses[1]
mark_summary_provider_completed()
_exception_raised = False
try:

View file

@ -0,0 +1,278 @@
from __future__ import annotations
import asyncio
import json
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, Protocol, runtime_checkable
import httpx
from fastapi import Request
from pydantic import JsonValue, TypeAdapter
from starlette.types import ASGIApp
import litellm
from litellm._uuid import uuid
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.internal_call_metadata import (
effective_turn_off_message_logging,
parent_session_kwargs,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
RequestRateLimiterStash,
finish_summary_cleanup,
get_request_stash,
isolated_summary_request_stash,
)
from litellm.proxy.spend_tracking.budget_reservation import (
release_budget_reservation_on_cancel, # pyright: ignore[reportUnknownVariableType] # legacy reservation dict is untyped
)
from litellm.router_strategy.complexity_router.context_compaction import NativeRequest
from litellm.types.utils import CONTEXT_COMPACTION_CALL_ORIGIN
_ASGI_APP: Final[TypeAdapter[ASGIApp]] = TypeAdapter(ASGIApp)
_STRING: Final = TypeAdapter(str)
_JSON_BODY: Final = TypeAdapter(Mapping[str, JsonValue])
_HEADERS: Final = TypeAdapter(Mapping[str, str])
_METADATA: Final = TypeAdapter(Mapping[str, object])
_NATIVE_BODY_FIELDS: Final = frozenset(
{"input", "instructions", "messages", "max_tokens", "system", "tools", "compaction"}
)
_OMITTED_HEADERS: Final = frozenset(
{
"content-length",
"content-type",
"transfer-encoding",
"x-litellm-call-id",
"x-app",
"x-claude-code-agent-id",
"x-litellm-model",
"x-litellm-model-id",
"litellm-disable-message-redaction",
}
)
@runtime_checkable
class SummaryRequestLimiter(Protocol):
async def release_summary_request_capacity(
self, auth: UserAPIKeyAuth, request_data: Mapping[str, object], provider_completed: bool
) -> None: ...
@dataclass(slots=True)
class _SummaryRequest:
call_id: str
redact: bool | None
auth: UserAPIKeyAuth | None = None
data: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
provider_completed: bool = False
closed: asyncio.Event = field(default_factory=asyncio.Event)
def bind(self, auth: UserAPIKeyAuth, data: Mapping[str, object]) -> None:
self.auth = auth
self.data = data
if (
self.redact is True
or effective_turn_off_message_logging(data) is True
or litellm.turn_off_message_logging is True
):
self.redact = True
_summary_request: Final[ContextVar[_SummaryRequest | None]] = ContextVar(
"litellm_context_compaction_request", default=None
)
def summary_request_updates(
request: Request,
auth: UserAPIKeyAuth,
data: Mapping[str, object],
metadata_key: str,
) -> Mapping[str, object]:
summary: Final = _summary_request.get()
if summary is None or summary.closed.is_set() or request.headers.get("x-litellm-call-id") != summary.call_id:
return MappingProxyType({})
summary.bind(auth, data)
metadata: Final = _METADATA.validate_python(data.get(metadata_key) or MappingProxyType({}))
stamped_metadata: Final = _METADATA.validate_python(
MappingProxyType({**metadata, INTERNAL_CALL_ORIGIN_METADATA_KEY: CONTEXT_COMPACTION_CALL_ORIGIN})
)
return MappingProxyType(
{
name: value
for name, value in (
(metadata_key, stamped_metadata),
("num_retries", 0),
("max_retries", 0),
("disable_fallbacks", True),
("turn_off_message_logging", True if summary.redact is True else None),
)
if value is not None
}
)
def mark_summary_provider_completed() -> None:
summary: Final = _summary_request.get()
if summary is not None and not summary.closed.is_set():
summary.provider_completed = True
def summary_request_is_private() -> bool:
summary: Final = _summary_request.get()
return summary is not None and not summary.closed.is_set() and summary.redact is True
async def _settle_cancelled_summary(
summary: _SummaryRequest,
limiter: SummaryRequestLimiter | None,
) -> None:
auth: Final = summary.auth
if auth is None:
return
if not summary.provider_completed:
await release_budget_reservation_on_cancel(auth.budget_reservation)
if limiter is not None:
await limiter.release_summary_request_capacity(auth, summary.data, summary.provider_completed)
@dataclass(frozen=True, slots=True)
class ProxySummaryExecutor:
request: Request
request_data: Mapping[str, object]
limiter: SummaryRequestLimiter | None
parent_stash: RequestRateLimiterStash | None
closed: asyncio.Event = field(default_factory=asyncio.Event)
call_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
async def __call__(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
if self.closed.is_set() or _summary_request.get() is not None:
raise ValueError("Context compaction executor is closed or nested")
return await asyncio.wait_for(self._serialized_execute(model, request, timeout), timeout)
async def _serialized_execute(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
async with self.call_lock:
if self.closed.is_set():
raise ValueError("Context compaction executor is closed")
return await self._execute(model, request, timeout)
async def _execute(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
dynamic_redact: Final = effective_turn_off_message_logging(self.request_data)
redact: Final = True if litellm.turn_off_message_logging is True else dynamic_redact
summary: Final = _SummaryRequest(call_id=str(uuid.uuid4()), redact=redact)
parent_call_id: Final = self.request_data.get("litellm_call_id")
scope: Final = _METADATA.validate_python(self.request.scope)
root_path: Final = _STRING.validate_python(scope.get("root_path", "")).rstrip("/")
endpoint: Final = "/v1/responses/compact" if request.protocol == "responses" else "/v1/messages"
url: Final = str(self.request.url.replace(path=f"{root_path}{endpoint}"))
native_headers: Final = httpx.Headers(
_HEADERS.validate_python(request.payload.get("extra_headers") or MappingProxyType({}))
)
beta: Final = ",".join(
dict.fromkeys(
token.strip()
for value in (self.request.headers.get("anthropic-beta"), native_headers.get("anthropic-beta"))
if value
for token in value.split(",")
if token.strip()
)
)
headers: Final = httpx.Headers(
tuple(
(name, value)
for name, value in self.request.headers.items()
if name.lower() not in _OMITTED_HEADERS and name.lower() != "anthropic-beta"
)
+ ((("anthropic-beta", beta),) if beta else ())
+ (("x-litellm-call-id", summary.call_id), ("content-type", "application/json"))
)
session: Final = parent_session_kwargs(self.request_data)
region: Final = self.request_data.get("allowed_model_region")
user: Final = self.request_data.get("user")
payload: Final = MappingProxyType(
{
**MappingProxyType(
{name: value for name, value in request.payload.items() if name in _NATIVE_BODY_FIELDS}
),
"allowed_model_region": region if isinstance(region, str) else None,
"user": user if isinstance(user, str) else None,
"litellm_session_id": session.get("litellm_session_id"),
"litellm_trace_id": session.get("litellm_trace_id"),
"model": model,
"timeout": timeout,
"stream": False if request.protocol == "messages" else None,
"num_retries": 0,
"max_retries": 0,
"disable_fallbacks": True,
"metadata": MappingProxyType({"tags": tuple(get_tags_from_request_body(self.request_data))}),
}
)
body: Final = json.dumps(
MappingProxyType({name: value for name, value in payload.items() if value is not None}),
default=dict,
allow_nan=False,
).encode()
address: Final = self.request.client
transport: Final = httpx.ASGITransport(
app=_ASGI_APP.validate_python(scope["app"]),
root_path=root_path,
client=(address.host, address.port) if address is not None else ("127.0.0.1", 0),
raise_app_exceptions=False,
)
async with isolated_summary_request_stash(
self.parent_stash, parent_call_id if isinstance(parent_call_id, str) else None, summary.call_id
):
token: Final = _summary_request.set(summary)
try:
async with transport:
response: Final = await transport.handle_async_request(
httpx.Request("POST", url, headers=headers, content=body)
)
try:
if not response.is_success:
raise litellm.APIError(
status_code=response.status_code,
message="Context compaction request failed",
model=model,
llm_provider="litellm",
)
return _JSON_BODY.validate_json(await response.aread())
finally:
await response.aclose()
except (asyncio.CancelledError, TimeoutError):
await finish_summary_cleanup(_settle_cancelled_summary(summary, self.limiter))
raise
finally:
summary.closed.set()
_summary_request.reset(token)
@contextmanager
def proxy_summary_executor_scope(
request: Request,
request_data: Mapping[str, object],
limiter: object,
) -> Generator[None]:
from litellm.router_strategy.complexity_router.context_compaction import use_summary_executor
if _summary_request.get() is not None:
yield
return
executor: Final = ProxySummaryExecutor(
request=request,
request_data=request_data,
limiter=limiter if isinstance(limiter, SummaryRequestLimiter) else None,
parent_stash=get_request_stash(),
)
try:
with use_summary_executor(executor):
yield
finally:
executor.closed.set()

View file

@ -10,7 +10,7 @@ import logging
import os
import uuid
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set
from contextlib import asynccontextmanager
from contextlib import AsyncExitStack, asynccontextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from datetime import datetime
@ -66,6 +66,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
from litellm.types.utils import (
CONTEXT_COMPACTION_CALL_ORIGIN,
CallTypes,
EmbeddingResponse,
ModelResponse,
@ -428,8 +429,8 @@ class ParallelRequestGauge(TypedDict):
class ParallelSlotAcquisition(TypedDict):
slot_id: str
counter_keys: list[str]
slot_id: ReadOnly[str]
counter_keys: ReadOnly[Sequence[str]]
class RateLimitStatus(TypedDict):
@ -559,6 +560,76 @@ def get_request_stash() -> RequestRateLimiterStash | None:
return _request_stash.get()
@dataclass(frozen=True, slots=True)
class _BorrowedParallelSlots:
stash: RequestRateLimiterStash
call_id: str
counter_keys: frozenset[str]
consumed: asyncio.Event = field(default_factory=asyncio.Event)
_borrowed_parallel_slots: Final[ContextVar[_BorrowedParallelSlots | None]] = ContextVar(
"litellm_summary_borrowed_parallel_slots", default=None
)
@asynccontextmanager
async def isolated_summary_request_stash(
parent_stash: RequestRateLimiterStash | None,
parent_call_id: str | None,
child_call_id: str,
) -> AsyncGenerator[RequestRateLimiterStash, None]:
async with AsyncExitStack() as stack:
eligible_parent: Final = (
parent_stash
if parent_stash is not None
and parent_call_id is not None
and parent_stash.owner_litellm_call_id == parent_call_id
and _borrowed_parallel_slots.get() is None
else None
)
if eligible_parent is not None:
await stack.enter_async_context(eligible_parent.parallel_slot_release_lock)
acquisition: Final = eligible_parent.parallel_slot if eligible_parent is not None else None
stash: Final = RequestRateLimiterStash(owner_litellm_call_id=child_call_id)
stash_token: Final = _request_stash.set(stash)
borrow_token: Final = _borrowed_parallel_slots.set(
_BorrowedParallelSlots(
stash=stash,
call_id=child_call_id,
counter_keys=frozenset(acquisition["counter_keys"]) if acquisition is not None else frozenset(),
)
)
try:
yield stash
finally:
_borrowed_parallel_slots.reset(borrow_token)
_request_stash.reset(stash_token)
def _consume_borrowed_parallel_keys(stash: RequestRateLimiterStash) -> frozenset[str]:
borrowed: Final = _borrowed_parallel_slots.get()
if (
borrowed is None
or borrowed.stash is not stash
or borrowed.call_id != stash.owner_litellm_call_id
or borrowed.consumed.is_set()
):
return frozenset()
borrowed.consumed.set()
return borrowed.counter_keys
async def finish_summary_cleanup(cleanup: Awaitable[None]) -> None:
task: Final = asyncio.ensure_future(cleanup)
while not task.done():
try:
await asyncio.shield(task)
except asyncio.CancelledError:
continue
task.result()
def get_or_create_request_stash() -> RequestRateLimiterStash:
stash = _request_stash.get()
if stash is None:
@ -1262,6 +1333,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
read_only: bool = False,
skip_tpm_check: bool = False,
parallel_slot_id: str | None = None,
borrowed_parallel_keys: frozenset[str] = frozenset(),
) -> RateLimitResponse:
"""
Check if any of the rate limit descriptors should be rate limited.
@ -1297,6 +1369,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
keys_to_fetch, key_metadata, gauges = self._collect_windowed_keys_and_gauges(
descriptors=descriptors,
skip_tpm_check=skip_tpm_check,
borrowed_parallel_keys=borrowed_parallel_keys,
)
windowed_response = RateLimitResponse(overall_code="OK", statuses=[])
@ -1385,6 +1458,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self,
descriptors: Sequence[RateLimitDescriptor],
skip_tpm_check: bool,
borrowed_parallel_keys: frozenset[str] = frozenset(),
) -> tuple[list[str], dict[str, WindowKeyMetadata], list[ParallelRequestGauge]]:
"""
Split descriptors into the windowed (window_key, counter_key) fetch
@ -1407,7 +1481,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
window_key = f"{{{descriptor_key}:{descriptor_value}}}:window"
if max_parallel_requests_limit is not None:
if (
max_parallel_requests_limit is not None
and self.create_rate_limit_keys(descriptor_key, descriptor_value, "max_parallel_requests")
not in borrowed_parallel_keys
):
gauges.append(
ParallelRequestGauge(
counter_key=self.create_rate_limit_keys(
@ -3492,6 +3570,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if response["overall_code"] == "OK":
await self._release_parallel_request_slots(acquisition, user_api_key_dict.parent_otel_span)
async def _admit_request_capacity(
self,
descriptors: Sequence[RateLimitDescriptor],
auth: UserAPIKeyAuth,
slot_id: str | None,
counter_keys: Sequence[str],
borrowed_parallel_keys: frozenset[str],
) -> RateLimitResponse:
admission: Final = self.should_rate_limit(
descriptors=descriptors,
parent_otel_span=auth.parent_otel_span,
skip_tpm_check=self.tpm_reservation_enabled,
parallel_slot_id=slot_id,
borrowed_parallel_keys=borrowed_parallel_keys,
)
if _borrowed_parallel_slots.get() is None:
return await admission
task: Final = asyncio.create_task(admission)
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
acquisition: Final = ParallelSlotAcquisition(slot_id=slot_id or "", counter_keys=tuple(counter_keys))
await finish_summary_cleanup(self._release_request_capacity_when_admitted(task, acquisition, auth))
raise
@asynccontextmanager
async def request_capacity(
self,
@ -3552,6 +3655,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook")
stash: Final = claim_request_stash_for_data(data)
borrowed_parallel_keys: Final = _consume_borrowed_parallel_keys(stash)
#########################################################
# Check if the call type has a specific rate limiter
@ -3595,6 +3699,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
for d in descriptors
if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None
and self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
not in borrowed_parallel_keys
]
parallel_slot_id: Final = uuid.uuid4().hex if parallel_counter_keys else None
@ -3605,11 +3711,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
d for d in descriptors if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
)
)
response: Final = await self.should_rate_limit(
response: Final = await self._admit_request_capacity(
descriptors=first_pass_descriptors,
parent_otel_span=user_api_key_dict.parent_otel_span,
skip_tpm_check=self.tpm_reservation_enabled,
parallel_slot_id=parallel_slot_id,
auth=user_api_key_dict,
slot_id=parallel_slot_id,
counter_keys=parallel_counter_keys,
borrowed_parallel_keys=borrowed_parallel_keys,
)
if response["overall_code"] == "OVER_LIMIT":
@ -4429,7 +4536,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# 'metadata' and 'litellm_metadata' fields from litellm_params
standard_logging_object: Final = kwargs.get("standard_logging_object") or {}
request_metadata: Final = get_litellm_metadata_from_kwargs(kwargs)
if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY):
call_origin: Final = request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)
if call_origin and call_origin != CONTEXT_COMPACTION_CALL_ORIGIN:
# Internal sub-calls bill spend to the caller but are not the caller's
# traffic; charging them here would let background evals eat TPM headroom.
return []
@ -4811,9 +4919,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if popped is not None:
await self.batch_enqueued_token_store.refund(reservation=popped, litellm_parent_otel_span=span)
async def release_summary_request_capacity(
self, auth: UserAPIKeyAuth, request_data: Mapping[str, object], provider_completed: bool
) -> None:
await self._release_stashed_parallel_slot(get_request_stash(), auth.parent_otel_span)
if not provider_completed:
await self.async_post_call_failure_hook(
request_data=request_data,
original_exception=TimeoutError("Context compaction summary cancelled"),
user_api_key_dict=auth,
)
async def async_post_call_failure_hook(
self,
request_data: dict,
request_data: Mapping[str, object],
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: str | None = None,

View file

@ -2194,6 +2194,10 @@ async def add_litellm_data_to_request(
if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True:
_strip_client_message_redaction_opt_out(data)
from litellm.proxy.common_utils.context_compaction import summary_request_is_private, summary_request_updates
data.update(summary_request_updates(request, user_api_key_dict, data, _metadata_variable_name))
# Fill in the proxy_server_request body snapshot now that metadata has
# been parsed. Consumers (standard_logging_payload, lago,
# spend_tracking_utils, streaming_iterator) read `body` to audit the
@ -2491,7 +2495,8 @@ async def add_litellm_data_to_request(
user_api_key_dict=user_api_key_dict,
)
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
if not summary_request_is_private():
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
# Team/Project credential overrides from model_config
# Placed after the debug log to avoid leaking credential secrets in logs

View file

@ -118,6 +118,21 @@ from litellm.llms.openai_like.model_info import (
get_openai_compatible_model_info,
)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.complexity_router.context_compaction import (
COMPACTION_STATE_KEY,
CompactedRequest,
CompactionFailure,
CompactionState,
ModelBudget,
NativeRequest,
compaction_state,
current_native_request,
current_summary_executor,
defers_context_filter,
prepare_compaction,
raise_compaction_failure,
validate_native_recipient,
)
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
@ -459,10 +474,19 @@ def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
def _silent_experiment_kwargs_snapshot(kwargs: Mapping[str, object]) -> Mapping[str, object]:
snapshot: Final = MappingProxyType({key: value for key, value in kwargs.items() if key != "messages"})
metadata: Final = kwargs.get("metadata")
if not isinstance(metadata, Mapping):
return MappingProxyType({**kwargs})
return MappingProxyType({**kwargs, "metadata": dict(metadata)})
return snapshot
return MappingProxyType({**snapshot, "metadata": TypeAdapter(dict[str, object]).validate_python(metadata)})
def _compaction_deployment_id(deployment: DeploymentTypedDict) -> str:
info: Final = TypeAdapter(Mapping[str, object]).validate_python(
deployment.get("model_info") or MappingProxyType({})
)
identifier: Final = info.get("id")
return identifier if isinstance(identifier, str) else ""
def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]:
@ -2617,7 +2641,7 @@ class Router:
return silent_kwargs
async def _run_silent_experiment(
self, silent_model: str, messages: Sequence[Mapping[str, str]], silent_kwargs: Mapping[str, object]
self, silent_model: str, messages: Sequence[Mapping[str, object]], silent_kwargs: Mapping[str, object]
) -> None:
remaining_kwargs: Final = MappingProxyType(
{key: value for key, value in silent_kwargs.items() if key != "stream"}
@ -2709,6 +2733,7 @@ class Router:
kwargs["messages"] = messages
kwargs["stream"] = stream
kwargs["original_function"] = self._acompletion
kwargs[COMPACTION_STATE_KEY] = CompactionState() # rebind-ok: share request-local state across retries
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs)
request_priority: Final = kwargs.get("priority") or self.default_priority
@ -3532,7 +3557,9 @@ class Router:
wrapper_ref: Final = weakref.ref(wrapped_response)
return wrapped_response
async def _silent_experiment_acompletion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
async def _silent_experiment_acompletion(
self, silent_model: str, messages: Sequence[Mapping[str, object]], **kwargs
):
"""
Run a silent experiment in the background.
"""
@ -3572,12 +3599,30 @@ class Router:
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
start_time: Final = time.time()
deployment = await self.async_get_available_deployment(
selected_deployment: Final = await self.async_get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
compacted_request: Final = await self._aprepare_selected_deployment(
deployment=cast( # cast-ok: Router registration validates deployments
DeploymentTypedDict, selected_deployment
),
model=model,
request_kwargs=kwargs,
messages=messages,
protocol="chat",
)
deployment = selected_deployment # rebind-ok: only prepared deployments count as provider attempts
kwargs.pop(COMPACTION_STATE_KEY, None)
effective_messages: Final = (
TypeAdapter(list[dict[str, object]]).validate_python(compacted_request.value)
if compacted_request is not None
else messages
)
if compacted_request is not None:
kwargs[compacted_request.field] = effective_messages # rebind-ok: rewrite only this dispatch attempt
self._drop_unsupported_classifier_reasoning_effort(
deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment
model=model,
@ -3615,12 +3660,12 @@ class Router:
asyncio.create_task(
self._silent_experiment_acompletion(
silent_model=silent_target,
messages=messages, # Use messages instead of *args
messages=effective_messages,
**_silent_experiment_kwargs_snapshot(kwargs),
)
)
kwargs.setdefault("messages", messages)
kwargs.setdefault("messages", effective_messages)
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
kwargs.pop("silent_model", None) # Ensure it's not in kwargs either
@ -3634,7 +3679,7 @@ class Router:
input_kwargs: Final = {
**litellm_params,
"messages": messages,
"messages": effective_messages,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
@ -5259,6 +5304,10 @@ class Router:
kwargs["model"] = model
kwargs["original_generic_function"] = original_function
kwargs["original_function"] = self._ageneric_api_call_with_fallbacks_helper
if original_function in (litellm.aresponses, litellm.anthropic_messages):
kwargs[COMPACTION_STATE_KEY] = ( # rebind-ok: preserve trusted state across generic streaming retries
compaction_state(kwargs) or CompactionState()
)
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata")
verbose_router_logger.debug(
"Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs
@ -5318,13 +5367,34 @@ class Router:
try:
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
try:
deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init above
selected_deployment: Final = await self.async_get_available_deployment(
model=model,
request_kwargs=kwargs,
messages=kwargs.get("messages", None),
input=kwargs.get("input", None),
specific_deployment=kwargs.pop("specific_deployment", None),
)
compacted_request: Final = await self._aprepare_selected_deployment(
deployment=cast( # cast-ok: Router registration validates deployments
DeploymentTypedDict, selected_deployment
),
model=model,
request_kwargs=kwargs,
messages=kwargs.get("messages"),
input=kwargs.get("input"),
protocol="responses" if kwargs.get("input") is not None else "messages",
)
deployment = selected_deployment # rebind-ok: only prepared deployments count as provider attempts
kwargs.pop(COMPACTION_STATE_KEY, None)
if compacted_request is not None:
prepared_history: Final = TypeAdapter(list[dict[str, object]]).validate_python(
compacted_request.value
)
kwargs[compacted_request.field] = prepared_history # rebind-ok: rewrite only this dispatch attempt
if compacted_request.extra_headers:
kwargs["extra_headers"] = TypeAdapter(dict[str, str]).validate_python(
compacted_request.extra_headers
)
except Exception as e:
if passthrough_on_no_deployment:
return await original_generic_function(model=model, **kwargs)
@ -5441,6 +5511,7 @@ class Router:
# bucket, so the post-call carry-over below always has somewhere to read and write.
kwargs.setdefault("litellm_metadata", {}) # mutable-ok: shared bucket # rebind-ok: stamp must be readable here
kwargs[COMPACTION_STATE_KEY] = CompactionState() # rebind-ok: share trusted request state with the snapshot
fallback_kwargs: Final[dict[str, object]] = kwargs.copy()
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
@ -5764,6 +5835,7 @@ class Router:
# bucket, so the post-call carry-over below always has somewhere to read and write.
kwargs.setdefault("litellm_metadata", {}) # mutable-ok: shared bucket # rebind-ok: stamp must be readable here
kwargs[COMPACTION_STATE_KEY] = CompactionState() # rebind-ok: share trusted request state with the snapshot
fallback_kwargs: Final[dict[str, object]] = kwargs.copy() # mutable-ok: mutated below before re-entry
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
@ -12278,11 +12350,231 @@ class Router:
client = self.cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span)
return client
def _compaction_model_budget(
self,
deployment: DeploymentTypedDict,
request_defaults: Mapping[str, object] = MappingProxyType({}),
) -> ModelBudget | CompactionFailure:
try:
info: Final = self.get_router_model_info(
deployment=TypeAdapter(dict[str, object]).validate_python(deployment),
received_model_name=deployment["model_name"],
)
except Exception: # noqa: BLE001 # model-map resolution failures are explicit compaction failures
return CompactionFailure("Context compaction requires known token limits for the selected deployment")
input_limit: Final = info.get("max_input_tokens")
output_limit: Final = info.get("max_output_tokens")
if type(input_limit) is not int or type(output_limit) is not int or min(input_limit, output_limit) <= 0:
return CompactionFailure(
"Context compaction requires positive input and output limits for the selected deployment"
)
wire_model: Final = deployment["litellm_params"].get("model")
if not isinstance(wire_model, str):
return CompactionFailure("Context compaction requires a concrete deployment model")
return ModelBudget(
wire_model,
input_limit,
output_limit,
request_defaults=request_defaults,
deployment_id=_compaction_deployment_id(deployment),
)
async def _acount_compaction_tokens(self, model: str, payload: Mapping[str, object]) -> int:
messages: Final = payload.get("messages")
input_value: Final = payload.get("input")
instructions: Final = payload.get("instructions")
return await offload_token_count(self._count_pre_call_check_tokens)(
model=model,
messages=TypeAdapter(list[dict[str, object]]).validate_python(
(
*(
(MappingProxyType({"role": "system", "content": instructions}),)
if isinstance(instructions, str) and instructions
else ()
),
*messages,
)
)
if isinstance(messages, (list, tuple))
else None,
input=TypeAdapter(str | list[object]).validate_python(input_value)
if isinstance(input_value, (str, list))
else None,
request_kwargs=payload,
)
def _compaction_request_payload(
self,
deployment: DeploymentTypedDict,
request_kwargs: Mapping[str, object],
messages: Sequence[Mapping[str, object]] | None = None,
input: str | Sequence[object] | None = None,
) -> Mapping[str, object]:
deployment_params: Final = TypeAdapter(dict[str, object]).validate_python(deployment)
overridden_marker_keys: Final = self._forwarded_alias_marker_keys_the_deployment_sets(
deployment=deployment_params,
forwarded_keys=request_kwargs.get(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()),
)
effective_kwargs: Final = TypeAdapter(dict[str, object]).validate_python(
MappingProxyType(
{
**(MappingProxyType({"messages": messages}) if messages is not None else MappingProxyType({})),
**(MappingProxyType({"input": input}) if input is not None else MappingProxyType({})),
**MappingProxyType(
{key: value for key, value in request_kwargs.items() if key not in overridden_marker_keys}
),
"metadata": TypeAdapter(dict[str, object]).validate_python(
request_kwargs.get("metadata") or MappingProxyType({})
),
}
)
)
self._merge_tools_from_deployment(deployment=deployment_params, kwargs=effective_kwargs)
self._update_kwargs_with_default_litellm_params(kwargs=effective_kwargs)
return MappingProxyType({**deployment["litellm_params"], **effective_kwargs})
async def _aprepare_selected_deployment(
self,
deployment: DeploymentTypedDict,
model: str,
request_kwargs: Mapping[str, object],
messages: Sequence[Mapping[str, object]] | None = None,
input: str | Sequence[object] | None = None,
protocol: Literal["chat", "responses", "messages"] = "chat",
) -> CompactedRequest | None:
state: Final = compaction_state(request_kwargs)
native_request: Final = current_native_request()
if native_request is None and (state is None or state.model is None):
return None
payload: Final = self._compaction_request_payload(deployment, request_kwargs, messages, input)
timeout: Final = self._get_non_stream_timeout(
kwargs=TypeAdapter(dict[str, object]).validate_python(request_kwargs),
data=TypeAdapter(dict[str, object]).validate_python(deployment["litellm_params"]),
)
if state is not None and isinstance(timeout, (int, float)):
state.limit_timeout(float(timeout))
target: Final = self._compaction_model_budget(deployment, request_defaults=payload)
if isinstance(target, CompactionFailure):
raise_compaction_failure(target, model)
if native_request is not None:
native_failure: Final = await validate_native_recipient(
native_request, target, payload, float(timeout) if isinstance(timeout, (int, float)) else 120.0
)
if native_failure is not None:
raise_compaction_failure(native_failure, model)
return None
if state is None or state.model is None:
return None
from litellm.litellm_core_utils.internal_call_metadata import (
effective_turn_off_message_logging,
forwarded_internal_call_metadata,
parent_session_kwargs,
)
metadata_source: Final = MappingProxyType(
{
key: TypeAdapter(dict[str, object]).validate_python(request_kwargs[key])
for key in ("metadata", "litellm_metadata")
if isinstance(request_kwargs.get(key), Mapping)
}
)
summary_metadata: Final = MappingProxyType(
forwarded_internal_call_metadata(
TypeAdapter(dict[str, object]).validate_python(
get_litellm_metadata_from_kwargs(
TypeAdapter(dict[str, object]).validate_python(
MappingProxyType({"litellm_params": metadata_source})
)
)
),
"context_compaction",
)
)
summary_request: Final = MappingProxyType({"metadata": summary_metadata})
async def sdk_summary(model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
if request_kwargs.get("proxy_server_request") is not None:
raise_compaction_failure(
CompactionFailure("The authenticated compaction executor is unavailable"), model
)
native_kwargs: Final = MappingProxyType(
{
**request.payload,
**parent_session_kwargs(request_kwargs),
**MappingProxyType(
{key: request_kwargs[key] for key in ("allowed_model_region", "tags") if key in request_kwargs}
),
"model": model,
**(MappingProxyType({"stream": False}) if request.protocol == "messages" else MappingProxyType({})),
"timeout": timeout,
"num_retries": 0,
"max_retries": 0,
"disable_fallbacks": True,
"metadata": TypeAdapter(dict[str, object]).validate_python(summary_metadata),
"turn_off_message_logging": effective_turn_off_message_logging(request_kwargs),
}
)
response: Final[object] = (
await self.acompact_responses(**native_kwargs)
if request.protocol == "responses"
else await self.aanthropic_messages(**native_kwargs)
)
return TypeAdapter(dict[str, object]).validate_python(
response.model_dump(exclude_none=True) if isinstance(response, BaseModel) else response
)
candidates: Final = cast( # cast-ok: access-group owner only filters Router-validated deployments
tuple[DeploymentTypedDict, ...],
tuple(
self._filter_deployments_by_model_access_groups(
model=self._get_model_from_alias(state.model) or state.model,
healthy_deployments=TypeAdapter(list[object]).validate_python(
self.deployments_for_request(state.model, summary_request)
),
request_kwargs=TypeAdapter(dict[str, object]).validate_python(summary_request),
request_team_id=get_request_team_id(summary_request),
),
),
)
budgets: Final = tuple(
self._compaction_model_budget(
candidate,
request_defaults=self._compaction_request_payload(
candidate,
MappingProxyType(
{
"messages": (),
"stream": False,
"num_retries": 0,
"max_retries": 0,
"disable_fallbacks": True,
}
),
),
)
for candidate in candidates
if not self._is_strategy_marker_deployment(candidate)
)
summary_budgets: Final = tuple(budget for budget in budgets if isinstance(budget, ModelBudget))
prepared: Final = await prepare_compaction(
payload=payload,
target=target,
state=state,
summary_budgets=summary_budgets if len(summary_budgets) == len(candidates) else (),
executor=current_summary_executor() or sdk_summary,
counter=self._acount_compaction_tokens,
protocol=protocol,
)
if isinstance(prepared, CompactionFailure):
raise_compaction_failure(prepared, model)
return prepared
def _count_pre_call_check_tokens(
self,
messages: list[dict[str, str]] | None,
input: str | list | None,
messages: Sequence[Mapping[str, object]] | None,
input: str | Sequence[object] | None,
request_kwargs: Mapping[str, object] | None = None,
model: str = "",
) -> int:
"""
Count input tokens for context-window pre-call checks.
@ -12301,7 +12593,7 @@ class Router:
anthropic_system_to_openai_message,
)
extras: Final = request_kwargs if request_kwargs is not None else MappingProxyType({})
extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({})
raw_instructions: Final = extras.get("instructions")
instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None
raw_tools: Final = extras.get("tools")
@ -12313,7 +12605,7 @@ class Router:
system_message: Final = anthropic_system_to_openai_message(extras.get("system"))
if messages is not None:
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
return litellm.token_counter(messages=counted_messages, tools=tools)
return litellm.token_counter(model=model, messages=counted_messages, tools=tools)
if input is not None:
from openai.types.responses.response_create_params import ResponseInputParam
@ -12327,6 +12619,7 @@ class Router:
responses_api_request={"instructions": instructions} if instructions is not None else {},
)
return litellm.token_counter(
model=model,
messages=cast(list, input_messages), # cast-ok: transformed chat messages
tools=tools,
)
@ -12452,7 +12745,11 @@ class Router:
model_info = self.get_router_model_info(deployment=deployment, received_model_name=model)
max_input_tokens = model_info.get("max_input_tokens") if isinstance(model_info, dict) else None
if isinstance(max_input_tokens, int) and has_countable_input:
if (
isinstance(max_input_tokens, int)
and has_countable_input
and not defers_context_filter(request_kwargs)
):
if input_tokens is None:
if skip_inline_token_count:
return _returned_deployments
@ -12745,6 +13042,22 @@ class Router:
_access_group_filter_emptied_candidates = (
_pre_model_access_group_filter_len > 0 and len(healthy_deployments) == 0
)
native_request: Final = current_native_request()
if native_request is not None:
native_deployments: Final = cast( # cast-ok: normal admission filters Router-validated deployments
Sequence[DeploymentTypedDict], healthy_deployments
)
healthy_deployments = TypeAdapter(
list[dict[str, object]]
).validate_python( # rebind-ok: constrain normal admission
tuple(
deployment
for deployment in native_deployments
if _compaction_deployment_id(deployment) in native_request.allowed_deployment_ids
)
)
if not healthy_deployments:
raise_compaction_failure(CompactionFailure("No eligible native compactor remains available"), model)
if len(healthy_deployments) == 0:
# check if the user sent in a deployment name instead
@ -13174,6 +13487,23 @@ class Router:
and self.routing_strategy != "latency-based-routing"
and self.routing_strategy != "least-busy"
): # prevent regressions for other routing strategies, that don't have async get available deployments implemented.
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
legacy_strategy: Final = self._select_pre_routing_strategy(
model=self._get_model_from_alias(model) or model, request_kwargs=request_kwargs
)
if (
legacy_strategy is not None
and isinstance(legacy_strategy.strategy, ComplexityRouter)
and legacy_strategy.strategy.config.context_window_compaction_model is not None
):
raise_compaction_failure(
CompactionFailure(
"Context compaction is unsupported with legacy synchronous deployment selection; "
"use simple-shuffle or usage-based-routing-v2"
),
model,
)
return self.get_available_deployment(
model=model,
messages=messages,
@ -13704,6 +14034,16 @@ class Router:
selected_strategy: Final = self._select_pre_routing_strategy(
model=registered_model_name, request_kwargs=request_kwargs
)
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
context_state: Final = compaction_state(request_kwargs)
if (
context_state is not None
and selected_strategy is not None
and isinstance(selected_strategy.strategy, ComplexityRouter)
and selected_strategy.strategy.config.context_window_compaction_model is not None
):
context_state.arm(selected_strategy.strategy.config.context_window_compaction_model)
if selected_strategy is None:
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
self._stamp_or_clear_metadata_key(

View file

@ -362,6 +362,60 @@ response = litellm.completion(
## Special Behaviors
### Compact history on context overflow
Set `context_window_compaction_model` to an ordinary Router model group that supports native
compaction and has enough context for the input. It receives older conversation history, so choose
a recipient appropriate for that data. The group must already exist in `model_list`; another Auto
Router cannot serve as the compaction model
```yaml
model_list:
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
tiers:
SIMPLE: small-model
COMPLEX: large-model
context_window_compaction_model: conversation-summary
```
Here `small-model`, `large-model`, and `conversation-summary` refer to your configured model groups.
Omitting the setting or using `null` keeps existing behavior. Empty and whitespace-only names are
rejected. When configured, compaction takes precedence over `enable_context_window_escalation`
Supported overflow routes are asynchronous Responses with native OpenAI deployments and Messages
with native Anthropic deployments, including streaming answers. OpenAI uses `/responses/compact`;
Anthropic uses on-demand compaction with the `compact-2026-09-04` beta. The producer must support
that provider's compaction API. Native state is not translated between providers or APIs
The answering deployment, including its session pin, stays unchanged. Completed older history goes
to the native compactor; the newest actual user request and its active tool tail stay untouched.
OpenAI's complete returned window or Anthropic's exact signed block is preserved, then the target's
native token-count endpoint checks compatibility and fit before answering. Requests that already
fit use their original history. Unsupported overflow routes, including Chat Completions, fail
explicitly rather than falling back to a custom summary. Synchronous SDK calls and raw passthrough
are outside this feature. The legacy `usage-based-routing` selector is unsupported; use an
async-native selector such as `simple-shuffle` or `usage-based-routing-v2`
Native compaction can omit details. It adds latency and a separately billable model call. There is
no exact upfront cost promise, automatic cost comparison, or guarantee it costs less than choosing
a larger model. The caller needs normal access to the configured compactor; its call remains
subject to normal budgets, rate limits, deployment restrictions, and spend accounting
The input budget reserves the effective output limit and a safety margin. Omitting the output limit
conservatively reserves the deployment's maximum output capability. Only one compaction call is
allowed within 120 seconds, further bounded by the request timeout. Anthropic compaction requests
reserve 4,096 output tokens; OpenAI's compact endpoint does not accept an output-token cap. Retries
can reuse the native result within the same request. A delayed streaming fallback cannot start new
proxy compaction calls after the initial response scope closes
If there is no eligible compactor, no closed older prefix, unavailable `previous_response_id`
history, or incompatible or oversized native output, the request fails with an explicit context
error. Token-count failures never fall back to a local estimate for native-state validation.
Original caller history is retained on failure; nothing is silently truncated
### Modality-based capability routing
The classifier reads text alone, so a request carrying an image can classify cheap and land on a

View file

@ -3099,7 +3099,11 @@ class ComplexityRouter(CustomLogger):
resolved_messages: Sequence[Mapping[str, object]] | None,
request_kwargs: Mapping[str, object],
) -> _RequestContextFit:
if not self.config.enable_context_window_escalation or not resolved_messages:
if (
self.config.context_window_compaction_model is not None
or not self.config.enable_context_window_escalation
or not resolved_messages
):
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
(self.config.default_model,) if self.config.default_model else ()
@ -3125,7 +3129,11 @@ class ComplexityRouter(CustomLogger):
(the placement stands). Only a real tokenizer count ever moves a request, escalation
lands only on groups whose every deployment declares a fitting window, and a group
with no resolvable window is never moved on faith in either direction."""
if not self.config.enable_context_window_escalation or not resolved_messages:
if (
self.config.context_window_compaction_model is not None
or not self.config.enable_context_window_escalation
or not resolved_messages
):
return None
pools: Final = self._tier_pools()
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))

View file

@ -1254,6 +1254,20 @@ class ComplexityRouterConfig(BaseModel):
),
)
context_window_compaction_model: str | None = Field(
default=None,
min_length=1,
pattern=r"\S",
description=(
"Ordinary Router model group used for provider-native compaction when an asynchronous "
"Responses/OpenAI or Messages/Anthropic request exceeds its selected deployment's "
"input budget. The compactor must fit the input and its native output must fit the "
"unchanged answering deployment. Unsupported overflow routes fail explicitly. "
"Compaction takes precedence over context window escalation and incurs a separate "
"billable call under normal access and limits. Nested Auto Routers are not supported. "
"Omit or set null to keep existing routing behavior."
),
)
enable_context_window_escalation: bool = Field(
default=True,
description=(

View file

@ -0,0 +1,403 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import time
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, Literal, NoReturn, Protocol
import anyio
from pydantic import StrictStr, TypeAdapter
import litellm
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.prompt_templates.compaction import (
CompactionHistoryError,
NativeProtocol,
compaction_headers,
native_compaction_payload,
native_compaction_result,
split_native_history,
)
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.openai.responses.count_tokens.handler import OpenAICountTokensHandler
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import ProviderSpecificHeader
COMPACTION_STATE_KEY: Final = "_context_window_compaction_state"
MAX_COMPACTION_SECONDS: Final = 120.0
SUMMARY_OUTPUT_TOKENS: Final = 4096
_PROTECTED_BODY_FIELDS: Final = frozenset(
{
"model",
"messages",
"input",
"prompt",
"instructions",
"system",
"tools",
"functions",
"tool_choice",
"function_call",
"max_tokens",
"max_completion_tokens",
"max_output_tokens",
"thinking",
"reasoning",
"context_management",
"compaction",
"previous_response_id",
"conversation",
}
)
_HEADERS: Final = TypeAdapter(dict[StrictStr, StrictStr])
_COUNT_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None]] = (
TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None)
)
@dataclass(frozen=True, slots=True)
class CompactionFailure:
message: str
@dataclass(frozen=True, slots=True)
class ModelBudget:
model: str
input_limit: int
output_limit: int
request_defaults: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
deployment_id: str = ""
def input_budget(self, output_tokens: int) -> int:
return self.input_limit - output_tokens - max(64, self.input_limit // 100)
@dataclass(frozen=True, slots=True)
class CompactedRequest:
field: str
value: Sequence[Mapping[str, object]]
extra_headers: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)
class NativeRequest:
protocol: NativeProtocol
payload: Mapping[str, object]
allowed_deployment_ids: frozenset[str]
closed: asyncio.Event = field(default_factory=asyncio.Event)
@dataclass(frozen=True, slots=True)
class SummaryMemo:
identity: str
response: Mapping[str, object]
class SummaryExecutor(Protocol):
async def __call__(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]: ...
class TokenCounter(Protocol):
async def __call__(self, model: str, payload: Mapping[str, object]) -> int: ...
class NativeTokenCounter(Protocol):
async def __call__(
self, target: ModelBudget, payload: Mapping[str, object], protocol: NativeProtocol, timeout: float
) -> int: ...
_summary_executor: Final[ContextVar[SummaryExecutor | None]] = ContextVar("context_compaction_executor", default=None)
_native_request: Final[ContextVar[NativeRequest | None]] = ContextVar("native_compaction_request", default=None)
@contextmanager
def use_summary_executor(executor: SummaryExecutor) -> Generator[None]:
token: Final = _summary_executor.set(executor)
try:
yield
finally:
_summary_executor.reset(token)
def current_summary_executor() -> SummaryExecutor | None:
return _summary_executor.get()
@contextmanager
def native_request_scope(request: NativeRequest) -> Generator[None]:
token: Final = _native_request.set(request)
try:
yield
finally:
request.closed.set()
_native_request.reset(token)
def current_native_request() -> NativeRequest | None:
request: Final = _native_request.get()
return request if request is not None and not request.closed.is_set() else None
class CompactionState:
def __init__(self, timeout: float = MAX_COMPACTION_SECONDS) -> None:
self.model: str | None = None
self.deadline: float = time.monotonic() + min(timeout, MAX_COMPACTION_SECONDS)
self.calls: int = 0
self.memo: SummaryMemo | None = None
self.failure: CompactionFailure | None = None
def arm(self, model: str) -> None:
self.model = model
def remaining(self) -> float:
return max(0.0, self.deadline - time.monotonic())
def limit_timeout(self, timeout: float) -> None:
self.deadline = min(self.deadline, time.monotonic() + timeout)
def failed(self, failure: CompactionFailure) -> CompactionFailure:
self.failure = failure
return failure
async def compact(self, request: NativeRequest, identity: str, executor: SummaryExecutor) -> Mapping[str, object]:
if self.memo is not None and self.memo.identity == identity:
return self.memo.response
if self.model is None or self.calls or self.remaining() <= 0:
raise ValueError("Native compaction exhausted its call or time budget")
self.calls += 1
with native_request_scope(request), anyio.fail_after(self.remaining()):
response: Final = await executor(self.model, request, self.remaining())
self.memo = SummaryMemo(identity, response)
return response
def compaction_state(payload: Mapping[str, object] | None) -> CompactionState | None:
candidate: Final = payload.get(COMPACTION_STATE_KEY) if payload is not None else None
return candidate if isinstance(candidate, CompactionState) else None
def defers_context_filter(payload: Mapping[str, object] | None) -> bool:
state: Final = compaction_state(payload)
return state is not None and state.model is not None
def raise_compaction_failure(failure: CompactionFailure, model: str) -> NoReturn:
raise litellm.ContextWindowExceededError(message=failure.message, model=model, llm_provider="")
def output_reservation(payload: Mapping[str, object], budget: ModelBudget) -> int | CompactionFailure:
ceilings: Final = tuple(
payload[key]
for key in ("max_output_tokens", "max_completion_tokens", "max_tokens")
if payload.get(key) is not None
)
if any(
isinstance(value, bool) or not isinstance(value, int) or not 0 < value <= budget.output_limit
for value in ceilings
):
return CompactionFailure("Native compaction requires valid output-token limits for the selected deployment")
ceiling: Final = max(TypeAdapter(tuple[int, ...]).validate_python(ceilings), default=budget.output_limit)
thinking: Final = payload.get("thinking")
thinking_budget: Final = thinking.get("budget_tokens") if isinstance(thinking, Mapping) else None
if thinking_budget is not None and (
isinstance(thinking_budget, bool) or not isinstance(thinking_budget, int) or not 0 < thinking_budget < ceiling
):
return CompactionFailure("The selected output ceiling must exceed the thinking budget")
return ceiling
def validate_compaction_overrides(payload: Mapping[str, object]) -> CompactionFailure | None:
body: Final = payload.get("extra_body")
if body is not None and (not isinstance(body, Mapping) or not _PROTECTED_BODY_FIELDS.isdisjoint(body)):
return CompactionFailure(
"Native compaction does not allow extra_body to override history or token-budget fields"
)
return None
def native_provider(model: str) -> tuple[str, str]:
resolved: Final = litellm.get_llm_provider(model=model)
return resolved[0].removeprefix("responses/"), resolved[1]
def supports_native_protocol(model: str, protocol: str) -> bool:
provider: Final = native_provider(model)[1]
return (protocol == "responses" and provider == "openai") or (protocol == "messages" and provider == "anthropic")
def _credential(value: object) -> str | None:
if not isinstance(value, str):
return None
return get_secret_str(value.removeprefix("os.environ/")) if value.startswith("os.environ/") else value
async def count_native_tokens(
target: ModelBudget, payload: Mapping[str, object], protocol: NativeProtocol, timeout: float
) -> int:
model, provider = native_provider(target.model)
if (protocol, provider) not in (("responses", "openai"), ("messages", "anthropic")):
raise ValueError("Native token counting requires a matching native provider")
params: Final = target.request_defaults
if any(source.get("client") is not None for source in (params, payload)) or any(
source.get("custom_llm_provider") not in (None, provider) for source in (params, payload)
):
raise ValueError("Native token counting cannot verify a custom client or provider override")
api_key: Final = _credential(params.get("api_key"))
api_base: Final = _credential(params.get("api_base"))
try:
scoped_headers: Final = _HEADERS.validate_python(
ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=_COUNT_SCOPED_HEADERS.validate_python(
payload.get("provider_specific_header", params.get("provider_specific_header"))
),
custom_llm_provider=provider,
)
)
header_sources: Final = (
*(source.get(field) for field in ("headers", "extra_headers") for source in (params, payload)),
scoped_headers,
)
headers: Final = MappingProxyType(
{
name.lower(): value
for source in header_sources
for name, value in _HEADERS.validate_python(
source if source is not None else MappingProxyType({})
).items()
}
)
handler: Final = OpenAICountTokensHandler() if protocol == "responses" else AnthropicCountTokensHandler()
return await handler.count_native_tokens(
model=model, payload=payload, api_key=api_key, api_base=api_base, headers=headers, timeout=timeout
)
except ValueError:
raise ValueError("Provider-native token counting failed") from None
async def validate_native_recipient(
request: NativeRequest,
target: ModelBudget,
payload: Mapping[str, object],
timeout: float,
counter: NativeTokenCounter = count_native_tokens,
) -> CompactionFailure | None:
if target.deployment_id not in request.allowed_deployment_ids or not supports_native_protocol(
target.model, request.protocol
):
return CompactionFailure("Native compaction selected an incompatible deployment")
invalid: Final = validate_compaction_overrides(payload)
if invalid is not None:
return invalid
for key in ("input", "messages", "instructions", "system", "tools", "compaction"):
if payload.get(key) != request.payload.get(key):
return CompactionFailure("Compactor deployment defaults override the native request")
reserve: Final = SUMMARY_OUTPUT_TOKENS if request.protocol == "messages" else target.output_limit
if request.protocol == "messages" and payload.get("max_tokens") != SUMMARY_OUTPUT_TOKENS:
return CompactionFailure("Compactor deployment overrides the native output allowance")
count: Final = await counter(target, payload, request.protocol, timeout)
if type(count) is not int or not 0 <= count <= target.input_budget(reserve):
return CompactionFailure("The selected native compactor cannot fit the complete request")
return None
def _has_native_state(payload: Mapping[str, object]) -> bool:
items: Final = payload.get("input") or payload.get("messages")
if not isinstance(items, (list, tuple)):
return False
return any(
isinstance(item, Mapping)
and (
item.get("type") in ("compaction", "reasoning", "item_reference")
or isinstance(item.get("content"), (list, tuple))
and any(isinstance(block, Mapping) and block.get("type") == "compaction" for block in item["content"])
)
for item in items
)
async def prepare_compaction(
payload: Mapping[str, object],
target: ModelBudget,
state: CompactionState,
summary_budgets: tuple[ModelBudget, ...],
executor: SummaryExecutor,
counter: TokenCounter,
protocol: Literal["chat", "responses", "messages"] = "chat",
native_counter: NativeTokenCounter = count_native_tokens,
) -> CompactedRequest | CompactionFailure | None:
invalid: Final = validate_compaction_overrides(payload)
if invalid is not None:
return invalid
reserve: Final = output_reservation(payload, target)
if isinstance(reserve, CompactionFailure):
return reserve
input_budget: Final = target.input_budget(reserve)
try:
if not _has_native_state(payload) and await counter(target.model, payload) <= input_budget:
return None
if protocol == "chat" or not supports_native_protocol(target.model, protocol):
return CompactionFailure(
"Native compaction requires Responses/OpenAI or Messages/Anthropic; this route is unsupported"
)
if payload.get("previous_response_id") is not None or payload.get("conversation") is not None:
return CompactionFailure("Native overflow compaction requires the complete conversation history")
with anyio.fail_after(state.remaining()):
input_tokens: Final = await native_counter(target, payload, protocol, state.remaining())
if type(input_tokens) is not int or input_tokens < 0:
return CompactionFailure("Native token counting returned an invalid input count")
if input_tokens <= input_budget:
return None
history: Final = split_native_history(payload, protocol)
if isinstance(history, CompactionHistoryError):
return CompactionFailure(history.message)
eligible: Final = tuple(
budget
for budget in summary_budgets
if budget.deployment_id
and supports_native_protocol(budget.model, protocol)
and budget.output_limit >= SUMMARY_OUTPUT_TOKENS
and input_tokens
<= budget.input_budget(SUMMARY_OUTPUT_TOKENS if protocol == "messages" else budget.output_limit)
)
if not eligible:
return CompactionFailure("No compatible configured native compactor can fit the complete input")
body: Final = native_compaction_payload(history, payload, protocol)
request: Final = NativeRequest(protocol, body, frozenset(budget.deployment_id for budget in eligible))
identity: Final = hashlib.sha256(
json.dumps(
MappingProxyType(
{"model": state.model, "protocol": protocol, "payload": body, "tools": payload.get("tools")}
),
sort_keys=True,
default=dict,
).encode()
).hexdigest()
response: Final = await state.compact(request, identity, executor)
result: Final = native_compaction_result(response, history, protocol)
if isinstance(result, CompactionHistoryError):
return CompactionFailure(result.message)
headers: Final = (
compaction_headers(payload.get("extra_headers")) if protocol == "messages" else MappingProxyType({})
)
prepared: Final = MappingProxyType(
{
**payload,
history.field: result,
"extra_headers": headers or payload.get("extra_headers") or MappingProxyType({}),
}
)
final_count: Final = await native_counter(target, prepared, protocol, state.remaining())
if type(final_count) is not int or not 0 <= final_count <= input_budget:
return CompactionFailure("Native compaction output does not fit the pinned target")
return CompactedRequest(history.field, result, headers)
except Exception as error: # noqa: BLE001 # provider failures become sanitized context errors at this boundary
return state.failed(
CompactionFailure(f"Native compaction failed ({type(error).__name__}); original history retained")
)

View file

@ -24,7 +24,7 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding"]
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "compaction"]
@dataclass(frozen=True, slots=True)
@ -154,6 +154,7 @@ def strategy_router_dependencies(
dict.fromkeys(
tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier"))
+ _named(litellm_params.get("complexity_router_default_model"), "default")
+ _named(complexity.get("context_window_compaction_model"), "compaction")
+ (
_named(classifier.get("model"), "classifier")
if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES

View file

@ -389,6 +389,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = (
)
class AnthropicCompactionRequest(TypedDict):
type: ReadOnly[Literal["summarize"]]
instructions: ReadOnly[NotRequired[str]]
class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
max_tokens: int | None
metadata: AnthropicMetadata | dict | None
@ -410,6 +415,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
output_config: AnthropicOutputConfig | None # Configuration for Claude's output behavior
cache_control: dict[str, Any] | None # Automatic prompt caching
reasoning_effort: str | None
compaction: ReadOnly[AnthropicCompactionRequest | None]
class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False):
@ -568,6 +574,7 @@ class CompactionBlock(TypedDict, total=False):
type: Required[Literal["compaction"]]
content: str | None
signature: ReadOnly[str]
class UsageIteration(TypedDict, total=False):

View file

@ -2954,6 +2954,7 @@ RoutingDecisionCause = Literal[
InternalCallOrigin = Literal[
"context_compaction",
"autorouter_classifier",
"shadow_eval_router",
"shadow_eval_judge",
@ -2964,6 +2965,7 @@ InternalCallOrigin = Literal[
records that it is not traffic the caller sent."""
AUTOROUTER_CLASSIFIER_CALL_ORIGIN: Final[InternalCallOrigin] = "autorouter_classifier"
CONTEXT_COMPACTION_CALL_ORIGIN: Final[InternalCallOrigin] = "context_compaction"
SHADOW_EVAL_ROUTER_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_router"
SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN: Final[InternalCallOrigin] = "llm_as_a_judge_guardrail"

View file

@ -0,0 +1,148 @@
import json
from copy import deepcopy
from typing import Final
import pytest
from litellm.litellm_core_utils.prompt_templates.compaction import (
CompactionHistoryError,
NativeHistory,
NativeProtocol,
compaction_headers,
native_compaction_payload,
native_compaction_result,
split_native_history,
)
_USER: Final = {"role": "user", "content": "latest"}
_HISTORY: Final = NativeHistory("input", (), (_USER,))
@pytest.mark.parametrize("protocol", ("responses", "messages"))
def test_native_payload_preserves_source_and_only_forwards_supported_parameters(protocol: NativeProtocol) -> None:
field: Final = "input" if protocol == "responses" else "messages"
older: Final = {"role": "user", "content": [{"type": "input_file", "file_id": "opaque-file"}]}
state: Final = {"type": "compaction", "encrypted_content": "opaque"}
payload: Final = {
field: [older, state, _USER],
"instructions": "rules",
"system": "system",
"tools": [{"name": "tool"}],
"temperature": 1,
"max_tokens": 1,
"extra_headers": {"Anthropic-Beta": "other", "x-test": "keep"},
}
original: Final = deepcopy(payload)
history: Final = split_native_history(payload, protocol)
assert isinstance(history, NativeHistory)
assert (history.field, history.prefix, history.tail) == (field, (older, state), (_USER,))
request: Final = native_compaction_payload(history, payload, protocol)
assert request[field] == [older, state]
if protocol == "responses":
assert request == {"input": [older, state], "instructions": "rules"}
else:
assert request == {
"messages": [older, state],
"system": "system",
"tools": [{"name": "tool"}],
"compaction": {"type": "summarize"},
"max_tokens": 4096,
"extra_headers": compaction_headers(payload["extra_headers"]),
}
request[field][0]["content"][0]["file_id"] = "changed-private-copy"
assert payload == original and history.prefix[0] == older
@pytest.mark.parametrize("mixed", (False, True))
def test_messages_tool_result_is_not_a_new_request(mixed: bool) -> None:
older: Final = {"role": "user", "content": "old"}
call: Final = {"role": "assistant", "content": [{"type": "tool_use", "id": "call"}]}
block: Final = {"type": "tool_result", "tool_use_id": "call", "content": "ok"}
result: Final = {"role": "user", "content": [block, *([{"type": "text", "text": "next"}] if mixed else [])]}
history: Final = split_native_history({"messages": [older, _USER, call, result]}, "messages")
assert isinstance(history, NativeHistory)
assert history.prefix == ((older, _USER, call) if mixed else (older,))
assert history.tail == ((result,) if mixed else (_USER, call, result))
@pytest.mark.parametrize("items", (None, [], ["invalid"], [{"role": "assistant", "content": "none"}], [_USER]))
@pytest.mark.parametrize("protocol", ("responses", "messages"))
def test_history_requires_older_prefix(items: object, protocol: NativeProtocol) -> None:
result: Final = split_native_history({"input" if protocol == "responses" else "messages": items}, protocol)
assert isinstance(result, CompactionHistoryError) and result.message
@pytest.mark.parametrize("field", ("previous_response_id", "conversation"))
def test_unresolved_history_rejected(field: str) -> None:
result: Final = split_native_history({"input": [_USER, _USER], field: "server-state"}, "responses")
assert isinstance(result, CompactionHistoryError) and "Server-side" in result.message
def test_responses_retains_canonical_window_and_only_removes_null_compaction_created_by() -> None:
retained: Final = {"type": "message", "role": "user", "content": "retained", "created_by": None}
compacted: Final = {"type": "compaction", "encrypted_content": " opaque ", "created_by": None, "other": None}
extra: Final = {**compacted, "created_by": "keep"}
unknown: Final = {"type": "future_item", "metadata": {"created_by": None}}
response: Final = {"output": [retained, compacted, extra, unknown]}
original: Final = deepcopy(response)
result: Final = native_compaction_result(response, _HISTORY, "responses")
assert result == (
retained,
{"type": "compaction", "encrypted_content": " opaque ", "other": None},
extra,
unknown,
_USER,
)
assert json.loads(json.dumps(result)) == [dict(item) for item in result]
assert response == original
def test_messages_preserves_signed_block_and_tail_without_aliasing() -> None:
block: Final = {"type": "compaction", "content": " Summary\n", "signature": "exact-signature", "extra": None}
response: Final = {"stop_reason": "compaction", "content": [block]}
original: Final = deepcopy(response)
result: Final = native_compaction_result(response, _HISTORY, "messages")
assert result == ({"role": "assistant", "content": [block]}, _USER)
assert json.loads(json.dumps(result)) == [dict(item) for item in result]
result[0]["content"][0]["signature"] = "changed-private-copy"
assert response == original
@pytest.mark.parametrize(
"protocol,field", (("responses", "encrypted_content"), ("messages", "signature"), ("messages", "content"))
)
@pytest.mark.parametrize("state", (None, "", " ", 42))
def test_invalid_compaction_state_rejected(protocol: NativeProtocol, field: str, state: object) -> None:
valid: Final = {"type": "compaction", "content": "summary", "signature": "signed", "encrypted_content": "opaque"}
block: Final = {**valid, field: state}
response: Final = {"stop_reason": "compaction", "content": [block], "output": [block]}
assert isinstance(native_compaction_result(response, _HISTORY, protocol), CompactionHistoryError)
@pytest.mark.parametrize("blocks", (None, [], ["invalid"], [{"type": "text", "text": "plain summary"}]))
@pytest.mark.parametrize("protocol", ("responses", "messages"))
def test_missing_native_output_rejected(blocks: object, protocol: NativeProtocol) -> None:
response: Final = {"stop_reason": "compaction", "content": blocks, "output": blocks}
assert isinstance(native_compaction_result(response, _HISTORY, protocol), CompactionHistoryError)
@pytest.mark.parametrize(("stop_reason", "count"), (("end_turn", 1), ("compaction", 2)))
def test_messages_requires_one_block_and_compaction_stop(stop_reason: str, count: int) -> None:
block: Final = {"type": "compaction", "content": "summary", "signature": "signed"}
response: Final = {"stop_reason": stop_reason, "content": [block] * count}
assert isinstance(native_compaction_result(response, _HISTORY, "messages"), CompactionHistoryError)
def test_beta_headers_merge_case_insensitively_and_idempotently() -> None:
required: Final = compaction_headers(None)["anthropic-beta"]
existing: Final = {"Anthropic-Beta": f"other, {required}", "anthropic-beta": "other,third", "x-test": "keep"}
original: Final = dict(existing)
merged: Final = compaction_headers(existing)
assert merged == {"anthropic-beta": f"other,{required},third", "x-test": "keep"}
assert compaction_headers(merged) == merged and existing == original
@pytest.mark.parametrize("headers", ([], {"x-test": 1}, {"x-test": b"bytes"}, {"anthropic-beta": None}))
def test_invalid_headers_fail_at_payload_boundary(headers: object) -> None:
with pytest.raises(ValueError, match="validation error"):
native_compaction_payload(_HISTORY, {"extra_headers": headers}, "messages")

View file

@ -328,8 +328,7 @@ class TestOpenAIResponsesAPIConfig:
"required": ["city"],
}
def test_transform_decodes_json_string_tool_parameters_on_compact_request(self):
"""The compact request path builds the same wire body, so it must decode too."""
def test_transform_compact_omits_json_string_tool_parameters(self):
_url, data = self.config.transform_compact_response_api_request(
model=self.model,
input="weather in Paris",
@ -341,7 +340,7 @@ class TestOpenAIResponsesAPIConfig:
headers={},
)
assert data["tools"][0]["parameters"] == {"type": "object"}
assert "tools" not in data
@pytest.mark.parametrize("raw_parameters", ['"just a string"', "not json at all", "[1, 2, 3]", 42])
def test_transform_rejects_tool_parameters_that_are_not_an_object(self, raw_parameters: object):
@ -386,13 +385,12 @@ class TestOpenAIResponsesAPIConfig:
assert "parameters" not in result["tools"][2]
assert result["tools"][3] == {"type": "web_search_preview"}
def test_transform_compact_drops_foreign_tool_call_item_ids(self):
"""The compact request path replays input the same way, so it must
apply the same id drop."""
@pytest.mark.parametrize("item_id, retained", [("toolu_01Foreign", False), ("fc_native", True)])
def test_transform_compact_sanitizes_tool_call_item_ids(self, item_id: str, retained: bool) -> None:
replayed_input = [
{
"type": "function_call",
"id": "toolu_01Foreign",
"id": item_id,
"call_id": "toolu_01Foreign",
"name": "get_weather",
"arguments": "{}",
@ -408,7 +406,9 @@ class TestOpenAIResponsesAPIConfig:
headers={},
)
assert "id" not in data["input"][0]
assert ("id" in data["input"][0]) is retained
if retained:
assert data["input"][0]["id"] == item_id
assert data["input"][0]["call_id"] == "toolu_01Foreign"
def test_transform_streaming_response(self):
@ -1915,7 +1915,7 @@ class TestFlattenToolSchemaCombinatorsWiring:
assert set(nested_parameters["properties"]) == {"id", "enabled", "schedule"}
assert json.loads(json.dumps(result["tools"])) == result["tools"]
def test_openai_compact_request_flattens_top_level_anyof(self):
def test_openai_compact_request_omits_anyof_tools(self):
_, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request(
model="gpt-4o",
input="hi",
@ -1925,7 +1925,7 @@ class TestFlattenToolSchemaCombinatorsWiring:
headers={},
)
assert "anyOf" not in data["tools"][0]["parameters"]
assert "tools" not in data
def test_openai_leaves_tools_without_rejected_keys_alone(self):
clean_tool = {
@ -2077,7 +2077,7 @@ class TestToolSchemaRegexPatternWiring:
assert result["tools"][0]["tools"][0]["parameters"]["properties"]["field"] == {"type": "string"}
def test_openai_compact_request_drops_patterns(self):
def test_openai_compact_request_omits_pattern_tools(self):
_, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request(
model="gpt-5.6",
input="hi",
@ -2087,7 +2087,7 @@ class TestToolSchemaRegexPatternWiring:
headers={},
)
assert data["tools"][0]["parameters"]["properties"]["field"] == {"type": "string"}
assert "tools" not in data
def test_non_openai_subclass_keeps_patterns(self):
from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig
@ -2208,3 +2208,34 @@ class TestReasoningFollowsModelSupport:
drop_params=True,
)
assert mapped["reasoning"] == {"effort": "medium"}
def test_native_compact_request_allowlist_preserves_input_and_caller_params() -> None:
from copy import deepcopy
from openai.types.responses import ResponseInputParam
model: Final = "native-compact-test"
native_input: Final[ResponseInputParam] = [
{"id": "cmp_test", "type": "compaction", "encrypted_content": "opaque"},
{"role": "user", "content": "latest question"},
]
allowed: Final = {
"instructions": "keep these rules", "previous_response_id": "resp_previous",
"prompt_cache_key": "cache-prefix", "prompt_cache_retention": "24h",
}
params: Final[dict[str, object]] = {
**allowed, "metadata": {"trace": "caller-owned"}, "stream": True,
"tools": [{"type": "function", "name": "lookup", "parameters": '{"type":"object"}'}],
"reasoning": {"effort": "medium"}, "max_output_tokens": 128,
}
original: Final = deepcopy((native_input, params))
_, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request(
model=model,
input=native_input,
response_api_optional_request_params=params,
api_base="https://api.openai.com/v1/responses",
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert data == {"model": model, "input": original[0], **allowed}
assert (native_input, params) == original

View file

@ -0,0 +1,233 @@
import asyncio
from collections.abc import Generator, Mapping
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from fastapi import FastAPI, Request
from pydantic import TypeAdapter
from starlette.types import ASGIApp
import litellm
from litellm import ModelResponse
from litellm.caching.caching import DualCache
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.context_compaction import (
_SummaryRequest,
_summary_request,
proxy_summary_executor_scope,
summary_request_updates,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
_PROXY_MaxParallelRequestsHandler_v3,
_request_stash,
get_request_stash,
)
from litellm.proxy.utils import InternalUsageCache, hash_token
from litellm.router_strategy.complexity_router.context_compaction import (
NativeRequest,
current_summary_executor,
)
_BODY: Final = TypeAdapter(dict[str, object])
_KEY: Final = "sk-test-context-compaction"
_KEY_HASH: Final = hash_token(_KEY)
_MESSAGES: Final = [{"role": "user", "content": "Private older conversation"}]
def parent_request(app: ASGIApp) -> Request:
return Request({
"type": "http", "method": "POST", "scheme": "https", "path": "/gateway/v1/messages",
"root_path": "/gateway", "query_string": b"", "server": ("gateway.test", 443),
"client": ("192.0.2.10", 2345), "app": app,
"headers": [(b"authorization", f"Bearer {_KEY}".encode()), (b"content-length", b"99999"),
(b"x-litellm-call-id", b"parent"), (b"litellm-disable-message-redaction", b"true"),
(b"x-app", b"cli"), (b"x-claude-code-agent-id", b"agent"),
(b"x-claude-code-session-id", b"session"), (b"x-litellm-model-id", b"target-id"),
(b"x-litellm-model", b"target"), (b"anthropic-beta", b"existing-beta")],
})
@pytest.fixture
def summary_proxy(
protocol: Literal["responses", "messages"], monkeypatch: pytest.MonkeyPatch,
) -> Generator[tuple[_PROXY_MaxParallelRequestsHandler_v3, AsyncMock], None, None]:
limiter: Final = _PROXY_MaxParallelRequestsHandler_v3(InternalUsageCache(DualCache()))
inference: Final = AsyncMock()
monkeypatch.setattr(litellm, "acompact_responses" if protocol == "responses" else "anthropic_messages", inference)
model: Final = "openai/gpt-5.6" if protocol == "responses" else "anthropic/claude-sonnet-5"
router: Final = litellm.Router(model_list=[{
"model_name": "summary", "litellm_params": {"model": model, "api_key": "test-only-key"},
}])
monkeypatch.setattr(proxy_server, "master_key", "sk-test-master")
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache())
monkeypatch.setattr(proxy_server.proxy_logging_obj, "max_parallel_request_limiter", limiter)
monkeypatch.setattr(litellm, "callbacks", [limiter])
token: Final = _request_stash.set(None)
try:
yield limiter, inference
finally:
_request_stash.reset(token)
@pytest.mark.asyncio
@pytest.mark.parametrize("outcome", ["success", "denied", "route-denied", "cancel", "timeout", "failed"])
@pytest.mark.parametrize("protocol", ["responses", "messages"])
async def test_real_proxy_summary_auth_identity_and_lifecycle(
outcome: Literal["success", "denied", "route-denied", "cancel", "timeout", "failed"], protocol: Literal["responses", "messages"],
summary_proxy: tuple[_PROXY_MaxParallelRequestsHandler_v3, AsyncMock],
) -> None:
limiter, inference = summary_proxy
native_payload: Final = (
{"input": _MESSAGES, "instructions": "Preserve the constraints"} if protocol == "responses" else
{"messages": _MESSAGES, "max_tokens": 128, "system": "Preserve the constraints", "tools": [],
"compaction": {"type": "summarize"}}
)
native: Final = NativeRequest(protocol, {
**native_payload, "model": "target", "api_key": "payload-credential", "extra_body": {"model": "target"},
"stream": True,
"extra_headers": {
"Anthropic-Beta": "existing-beta, compact-2026-09-04",
"authorization": "Bearer other", "x-api-key": "other",
},
}, frozenset({"native-test"}))
result: Final = {
"id": "native-result", "usage": {"input_tokens": 10, "output_tokens": 2},
**({"object": "response.compaction", "output": [{"type": "compaction", "encrypted_content": "opaque"}]}
if protocol == "responses" else {
"type": "message", "role": "assistant", "stop_reason": "compaction",
"content": [{"type": "compaction", "content": "opaque", "signature": "signed"}],
}),
}
auth: Final = UserAPIKeyAuth(
api_key=_KEY_HASH, models=["target"] if outcome == "denied" else ["target", "summary"],
max_parallel_requests=1, rpm_limit=10,
allowed_routes=["/v1/chat/completions"] if outcome == "route-denied" else ["llm_api_routes"],
)
await proxy_server.user_api_key_cache.async_set_cache(_KEY_HASH, auth, model_type=UserAPIKeyAuth)
parent_data: Final = {
"model": "target", "litellm_call_id": "parent", "turn_off_message_logging": True,
"allowed_model_region": "eu", "user": "end-user", "litellm_session_id": "session",
"metadata": {"tags": ["cost-center"]},
}
await limiter.async_pre_call_hook(auth, DualCache(), parent_data, "acompletion")
parent: Final = get_request_stash()
assert parent is not None and parent.parallel_slot is not None
acquisition: Final = parent.parallel_slot
entered: Final = asyncio.Event()
stopped: Final = asyncio.Event()
application: Final = AsyncMock(wraps=proxy_server.app.__call__)
async def receive(**provider_data: object) -> Mapping[str, object]:
data: Final = _BODY.validate_python(_BODY.validate_python(provider_data["proxy_server_request"])["body"])
assert get_request_stash() is not parent
assert parent.parallel_slot is acquisition
child: Final = Request(application.call_args.args[0])
assert child.url.path == "/gateway/v1/" + ("responses/compact" if protocol == "responses" else "messages")
assert child.headers["authorization"] == f"Bearer {_KEY}"
assert "x-api-key" not in child.headers
assert child.headers["anthropic-beta"] == "existing-beta,compact-2026-09-04"
if protocol == "responses":
assert "stream" not in provider_data
else:
assert provider_data["stream"] is False
assert child.client.host == "192.0.2.10"
assert not {"litellm-disable-message-redaction", "x-app", "x-claude-code-agent-id",
"x-litellm-model-id", "x-litellm-model"}.intersection(child.headers)
assert child.headers["x-claude-code-session-id"] == "session"
assert data["litellm_call_id"] != "parent"
for name, expected in (
("turn_off_message_logging", True), ("num_retries", 0), ("max_retries", 0),
("disable_fallbacks", True), ("allowed_model_region", "eu"), ("user", "end-user"),
("litellm_session_id", "session"), ("model", "summary"), *native_payload.items(),
):
assert data[name] == expected
assert provider_data.get("api_key") != "payload-credential"
assert "extra_body" not in data
metadata: Final = _BODY.validate_python(data["litellm_metadata"])
assert metadata[INTERNAL_CALL_ORIGIN_METADATA_KEY] == "context_compaction"
assert metadata["user_api_key"] == _KEY_HASH
assert "cost-center" in metadata["tags"]
with pytest.raises(ValueError, match="nested"):
await executor("summary", native, 10)
entered.set()
try:
if outcome in ("cancel", "timeout"):
await asyncio.Event().wait()
if outcome == "failed":
raise litellm.APIError(503, "Private older conversation", llm_provider="test", model="summary")
return result
finally:
stopped.set()
inference.side_effect = receive
assert current_summary_executor() is None
with proxy_summary_executor_scope(parent_request(application), parent_data, limiter):
executor: Final = current_summary_executor()
assert executor is not None
task: Final = asyncio.create_task(executor("summary", native, 0.2 if outcome == "timeout" else 10))
if outcome in ("denied", "route-denied", "failed"):
with pytest.raises(litellm.APIError) as denied:
await task
assert denied.value.status_code in ((503,) if outcome == "failed" else (401, 403))
assert "Private older conversation" not in str(denied.value)
assert inference.await_count == int(outcome == "failed")
cached: Final = await proxy_server.user_api_key_cache.async_get_cache(_KEY_HASH, model_type=UserAPIKeyAuth)
assert cached.models == auth.models and cached.spend == 0
else:
if outcome == "success":
assert await task == result
else:
await asyncio.wait_for(entered.wait(), 5)
if outcome == "cancel":
task.cancel()
with pytest.raises(asyncio.CancelledError if outcome == "cancel" else TimeoutError):
await task
inference.assert_awaited_once()
assert stopped.is_set()
assert current_summary_executor() is None
assert get_request_stash() is parent and parent.parallel_slot is acquisition
assert not parent.reservation_released
with pytest.raises(ValueError, match="closed"):
await executor("summary", native, 10)
await limiter.async_post_call_success_hook(parent_data, auth, ModelResponse())
assert parent.parallel_slot is None
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=proxy_server.app)) as client:
rejected: Final = await client.post(
"https://gateway.test/v1/" + ("responses/compact" if protocol == "responses" else "messages"),
headers={"authorization": f"Bearer {_KEY}"},
json={**native_payload, "model": "summary", "turn_off_message_logging": True},
)
assert rejected.is_client_error and "turn_off_message_logging" in rejected.text
assert proxy_server.general_settings == {}
@pytest.mark.parametrize("parent_redact", [False, True, None])
@pytest.mark.parametrize("recipient_redact", [False, True, None])
@pytest.mark.parametrize("global_redact", [False, True])
def test_summary_privacy_never_lowers_recipient_policy(
parent_redact: bool | None, recipient_redact: bool | None,
global_redact: bool, monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "turn_off_message_logging", global_redact)
summary: Final = _SummaryRequest(call_id="parent", redact=parent_redact)
token: Final = _summary_request.set(summary)
recipient_data: Final = {"turn_off_message_logging": recipient_redact, "metadata": {}}
try:
updates: Final = summary_request_updates(
parent_request(FastAPI()), UserAPIKeyAuth(), recipient_data, "metadata"
)
if parent_redact is True or recipient_redact is True or global_redact:
assert updates["turn_off_message_logging"] is True
assert summary.redact is True
else:
assert "turn_off_message_logging" not in updates
finally:
_summary_request.reset(token)

View file

@ -10,7 +10,7 @@ import time
from collections.abc import Sequence
from contextlib import contextmanager
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from typing import Any, Dict, Final, List, Optional
import pytest
from fastapi import HTTPException
@ -27,6 +27,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
_request_stash,
get_or_create_request_stash,
get_request_stash,
isolated_summary_request_stash,
)
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
_PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler,
@ -6066,6 +6067,104 @@ def test_internal_call_origin_success_ops_are_skipped():
assert charged
assert skipped == []
summary = handler._build_success_event_pipeline_operations(
kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: "context_compaction"}),
response_obj=response,
rate_limit_type="output",
)
assert summary == charged
@pytest.mark.asyncio
@pytest.mark.parametrize("parent_owner", ["parent", "unrelated", None])
async def test_summary_borrows_only_live_direct_parent_and_preserves_rpm(parent_owner):
cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(cache))
auth = UserAPIKeyAuth(api_key=hash_token("sk-summary"), max_parallel_requests=1, rpm_limit=2)
parent_data = {"model": "target", "litellm_call_id": "parent"}
await handler.async_pre_call_hook(auth, cache, parent_data, "acompletion")
parent = get_request_stash()
acquisition = parent.parallel_slot
counter_key = acquisition["counter_keys"][0]
child_data = {"model": "summary", "litellm_call_id": "child"}
async with isolated_summary_request_stash(parent, parent_owner, "child") as child:
assert child is not parent
if parent_owner == "parent":
await handler.async_pre_call_hook(auth, cache, child_data, "acompletion")
assert child.parallel_slot is None
await handler.async_post_call_success_hook(child_data, auth, ModelResponse())
with pytest.raises(HTTPException):
async with isolated_summary_request_stash(child, "child", "grandchild"):
await handler.async_pre_call_hook(
auth, cache, {"model": "summary", "litellm_call_id": "grandchild"}, "acompletion"
)
else:
with pytest.raises(HTTPException):
await handler.async_pre_call_hook(auth, cache, child_data, "acompletion")
assert parent.parallel_slot is acquisition
assert handler._gauge_in_flight_from_cache_value(await cache.async_get_cache(key=counter_key)) == 1
assert get_request_stash() is parent
await handler.async_post_call_success_hook(parent_data, auth, ModelResponse())
assert handler._gauge_in_flight_from_cache_value(await cache.async_get_cache(key=counter_key)) == 0
with pytest.raises(HTTPException):
await handler.async_pre_call_hook(auth, cache, {"model": "summary"}, "acompletion")
@pytest.mark.asyncio
async def test_summary_lifetime_keeps_parent_release_locked():
cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(cache))
auth = UserAPIKeyAuth(api_key=hash_token("sk-summary-lock"), max_parallel_requests=1)
data = {"model": "target", "litellm_call_id": "parent"}
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
parent = get_request_stash()
async with isolated_summary_request_stash(parent, "parent", "child"):
release = asyncio.create_task(handler._release_stashed_parallel_slot(parent, None))
await asyncio.sleep(0)
assert not release.done()
assert parent.parallel_slot is not None
await release
assert parent.parallel_slot is None
def test_summary_gauge_exclusion_retains_new_model_gauge_and_windowed_limits():
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(DualCache()))
descriptors = (
{"key": "api_key", "value": "caller", "rate_limit": {
"max_parallel_requests": 1, "requests_per_unit": 5, "tokens_per_unit": 100,
}},
{"key": "model_per_key", "value": "summary:caller", "rate_limit": {"max_parallel_requests": 1}},
)
all_keys, all_metadata, all_gauges = handler._collect_windowed_keys_and_gauges(descriptors, False)
keys, metadata, gauges = handler._collect_windowed_keys_and_gauges(
descriptors, False, frozenset({all_gauges[0]["counter_key"]})
)
assert keys == all_keys
assert metadata == all_metadata
assert gauges == [all_gauges[1]]
@pytest.mark.asyncio
async def test_summary_origin_reconciles_actual_key_and_team_tpm():
cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(cache))
key = hash_token("sk-summary-accounting")
stash = get_or_create_request_stash()
stash.owner_litellm_call_id = "summary-child"
stash.reserved_tokens = 40
stash.reserved_scopes = frozenset({("api_key", key), ("team", "team")})
for scope, value in stash.reserved_scopes:
await cache.async_set_cache(handler.create_rate_limit_keys(scope, value, "tokens"), 40, ttl=60)
response = ModelResponse(usage=Usage(prompt_tokens=7, completion_tokens=5, total_tokens=12))
await handler.async_log_success_event(
kwargs={
"litellm_call_id": "summary-child", "model": "summary",
"standard_logging_object": {"metadata": {"user_api_key_hash": key, "user_api_key_team_id": "team"}},
"litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: "context_compaction"}},
}, response_obj=response, start_time=None, end_time=None,
)
for scope, value in stash.reserved_scopes:
assert await cache.async_get_cache(handler.create_rate_limit_keys(scope, value, "tokens")) == 12
def _conflicting_budget_bodies() -> Dict[str, Dict[str, object]]:
@ -6609,6 +6708,58 @@ class _DelayedCapacityUsageCache:
await self.finish_release.wait()
@pytest.mark.asyncio
async def test_summary_admission_cancellation_releases_only_child_slots_despite_repeated_cancel():
cache: Final = _DelayedCapacityUsageCache()
parent_handler: Final = _PROXY_MaxParallelRequestsHandler(cache.delegate)
child_handler: Final = _PROXY_MaxParallelRequestsHandler(cache)
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-summary-cancel-admission"), max_parallel_requests=1)
parent_data: Final = {"model": "target", "litellm_call_id": "parent"}
await parent_handler.async_pre_call_hook(auth, cache.dual_cache, parent_data, "acompletion")
parent: Final = get_request_stash()
assert parent is not None and parent.parallel_slot is not None
parent_acquisition: Final = parent.parallel_slot
parent_key: Final = parent_acquisition["counter_keys"][0]
child_key: Final = child_handler.create_rate_limit_keys("model_per_key", "summary:caller", "max_parallel_requests")
descriptors: Final = (
{"key": "api_key", "value": auth.api_key, "rate_limit": {"max_parallel_requests": 1}},
{"key": "model_per_key", "value": "summary:caller", "rate_limit": {"max_parallel_requests": 1}},
)
async def admit_child() -> None:
async with isolated_summary_request_stash(parent, "parent", "child"):
await child_handler._admit_request_capacity(
descriptors, auth, "child-slot", (child_key,), frozenset(parent_acquisition["counter_keys"])
)
pytest.fail("cancelled summary admission entered provider body")
task: Final = asyncio.create_task(admit_child())
try:
await asyncio.wait_for(cache.acquired.wait(), timeout=2)
task.cancel()
await asyncio.sleep(0)
cache.finish_admission.set()
await asyncio.wait_for(cache.releasing.wait(), timeout=2)
task.cancel()
await asyncio.sleep(0)
task.cancel()
await asyncio.sleep(0)
assert not task.done()
cache.finish_release.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=2)
assert get_request_stash() is parent
assert parent.parallel_slot is parent_acquisition
assert parent_handler._gauge_in_flight_from_cache_value(await cache.dual_cache.async_get_cache(parent_key)) == 1
assert child_handler._gauge_in_flight_from_cache_value(await cache.dual_cache.async_get_cache(child_key)) == 0
finally:
cache.finish_admission.set()
cache.finish_release.set()
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await parent_handler.async_post_call_success_hook(parent_data, auth, ModelResponse())
@pytest.mark.asyncio
async def test_request_capacity_finishes_admission_and_release_despite_repeated_cancel():
cache = _DelayedCapacityUsageCache()

View file

@ -1,6 +1,6 @@
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
from typing import Final, Literal
import pytest
from fastapi import HTTPException
@ -10,6 +10,7 @@ from litellm.proxy._types import (
LiteLLM_TeamTable,
LitellmUserRoles,
Member,
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.management_helpers.auto_router_permissions import (
@ -120,6 +121,29 @@ def test_all_tier_parameter_representations_reject_privileged_overrides(
assert denied.value.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize("denied_scope", [None, "team", "key"])
async def test_compaction_recipient_requires_normal_model_access(
catalog: Router, denied_scope: Literal["team", "key"] | None
) -> None:
operation: Final = authorize_member_auto_router_dependencies(
config=validate_member_auto_router_config(
{"tiers": {"SIMPLE": "allowed"}, "context_window_compaction_model": "other"}
),
default_model=None,
user_api_key_dict=_actor(models=["allowed"] if denied_scope == "key" else ["allowed", "other"]),
team=_team(models=["allowed"] if denied_scope == "team" else ["allowed", "other"]),
prisma_client=_Client(),
llm_router=catalog,
)
if denied_scope is None:
await operation
return
with pytest.raises(ProxyException, match="other") as denied:
await operation
assert str(denied.value.code) == "403"
def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> None:
validated: Final = validate_member_auto_router_config(
{"tiers": {"SIMPLE": [{"model_name": "allowed", "litellm_params": {"reasoning_effort": "low"}}]}}
@ -183,7 +207,10 @@ async def test_member_updates_restrict_fields_and_preserve_an_inherited_default(
@pytest.mark.asyncio
@pytest.mark.parametrize("target", ["missing", "nested"])
async def test_member_dependencies_require_plain_configured_models(target: str) -> None:
@pytest.mark.parametrize("role", ["tier", "compaction"])
async def test_member_dependencies_require_plain_configured_models(
target: str, role: Literal["tier", "compaction"]
) -> None:
catalog: Final = Router(
model_list=[
{"model_name": "allowed", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}},
@ -198,10 +225,14 @@ async def test_member_dependencies_require_plain_configured_models(target: str)
)
with pytest.raises(HTTPException) as denied:
await authorize_member_auto_router_dependencies(
config=validate_member_auto_router_config({"tiers": {"SIMPLE": target}}),
config=validate_member_auto_router_config(
{"tiers": {"SIMPLE": target}}
if role == "tier"
else {"tiers": {"SIMPLE": "allowed"}, "context_window_compaction_model": target}
),
default_model=None,
user_api_key_dict=_actor(models=[target]),
team=_team(models=[target]),
user_api_key_dict=_actor(models=["allowed", target]),
team=_team(models=["allowed", target]),
prisma_client=_Client(),
llm_router=catalog,
)

View file

@ -0,0 +1,273 @@
import asyncio
from collections.abc import Mapping
from contextvars import Context, copy_context
from copy import deepcopy
from dataclasses import dataclass, field, replace
from queue import SimpleQueue
from typing import Final, Literal
import pytest
from litellm.litellm_core_utils.prompt_templates.compaction import NativeProtocol
from litellm.router_strategy.complexity_router import context_compaction as core
pytestmark: Final = pytest.mark.asyncio
def _budget(protocol: NativeProtocol, identifier: str = "target") -> core.ModelBudget:
provider: Final = "openai" if protocol == "responses" else "anthropic"
return core.ModelBudget(f"{provider}/native-test", 2_000, 256, deployment_id=identifier)
_HISTORY: Final = (
{"role": "user", "content": "old"},
{"role": "assistant", "content": "answer"},
{"role": "user", "content": "latest"},
)
def _payload(protocol: NativeProtocol) -> Mapping[str, object]:
return {
"input" if protocol == "responses" else "messages": list(_HISTORY),
"instructions" if protocol == "responses" else "system": "rules",
"tools": [],
}
@dataclass(frozen=True)
class _Rig:
counts: tuple[int | Exception, ...] = (2_000, 100)
local_count: int = 2_000
error: Exception | None = None
block: bool = False
calls: SimpleQueue[tuple[str, core.NativeRequest, Context]] = field(default_factory=SimpleQueue)
native: SimpleQueue[tuple[core.ModelBudget, Mapping[str, object], NativeProtocol]] = field(
default_factory=SimpleQueue
)
local: SimpleQueue[str] = field(default_factory=SimpleQueue)
started: asyncio.Event = field(default_factory=asyncio.Event)
async def counter(self, model: str, payload: Mapping[str, object]) -> int:
self.local.put(model)
return self.local_count
async def native_counter(
self, target: core.ModelBudget, payload: Mapping[str, object], protocol: NativeProtocol, timeout: float
) -> int:
assert timeout > 0
self.native.put((target, payload, protocol))
result: Final = self.counts[(self.native.qsize() - 1) % len(self.counts)]
if isinstance(result, Exception):
raise result
return result
async def execute(self, model: str, request: core.NativeRequest, timeout: float) -> Mapping[str, object]:
assert core.current_native_request() is request and timeout > 0
self.calls.put((model, request, copy_context()))
self.started.set()
if self.error is not None:
raise self.error
if self.block:
await asyncio.Event().wait()
if request.protocol == "responses":
return {"output": [{"type": "compaction", "encrypted_content": "opaque"}]}
return {
"stop_reason": "compaction",
"content": [{"type": "compaction", "content": "native context", "signature": "signed"}],
}
async def _run(
rig: _Rig,
protocol: NativeProtocol = "responses",
state: core.CompactionState | None = None,
payload: Mapping[str, object] | None = None,
budgets: tuple[core.ModelBudget, ...] | None = None,
target: core.ModelBudget | None = None,
route: Literal["chat", "responses", "messages"] | None = None,
) -> core.CompactedRequest | core.CompactionFailure | None:
active: Final = core.CompactionState() if state is None else state
active.arm(active.model or "compactors")
recipient: Final = _budget(protocol) if target is None else target
candidates: Final = (
(replace(recipient, input_limit=10_000, output_limit=core.SUMMARY_OUTPUT_TOKENS, deployment_id="allowed"),)
if budgets is None
else budgets
)
return await core.prepare_compaction(
_payload(protocol) if payload is None else payload,
recipient,
active,
candidates,
rig.execute,
rig.counter,
route or protocol,
rig.native_counter,
)
@pytest.mark.parametrize("protocol", ("responses", "messages"))
@pytest.mark.parametrize("mode", ("success", "error", "timeout", "cancel", "expired"))
async def test_candidates_recipient_and_context_cleanup(protocol: NativeProtocol, mode: str) -> None:
target: Final = _budget(protocol)
good: Final = replace(target, input_limit=10_000, output_limit=core.SUMMARY_OUTPUT_TOKENS, deployment_id="good")
bad: Final = (
replace(good, deployment_id="small", input_limit=1_000),
replace(good, deployment_id="output", output_limit=1),
replace(good, deployment_id=""),
replace(good, deployment_id="other", model="anthropic/test" if protocol == "responses" else "openai/test"),
)
rig: Final = _Rig(
error=RuntimeError("private native response") if mode == "error" else None, block=mode in ("timeout", "cancel")
)
state: Final = core.CompactionState(timeout=0 if mode == "expired" else 0.02 if mode == "timeout" else 10)
payload: Final = _payload(protocol)
original: Final = deepcopy(payload)
with core.use_summary_executor(rig.execute):
assert core.current_summary_executor() == rig.execute
task: Final = asyncio.create_task(_run(rig, protocol, state, payload, (*bad, good), target))
if mode == "cancel":
await rig.started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
else:
assert isinstance(await task, core.CompactedRequest if mode == "success" else core.CompactionFailure)
assert core.current_summary_executor() is None and core.current_native_request() is None and payload == original
assert rig.calls.qsize() == (0 if mode == "expired" else 1)
if mode != "expired":
model, request, inherited = rig.calls.get_nowait()
assert model == "compactors" and request.allowed_deployment_ids == frozenset({"good"})
assert request.closed.is_set() and inherited.run(core.current_native_request) is None
if mode == "success":
before, after = (rig.native.get_nowait() for _ in range(2))
result: Final = task.result()
assert isinstance(result, core.CompactedRequest) and after[1][result.field] == result.value
assert before[0] is target and after[0] is target and after[2] == protocol
denied: Final = _Rig()
assert isinstance(await _run(denied, protocol, budgets=bad), core.CompactionFailure) and denied.calls.empty()
@pytest.mark.parametrize("scenario", ("local_fit", "native_fit", "chat", "provider", "no_prefix"))
async def test_fit_and_unsupported_admission(scenario: str) -> None:
rig: Final = _Rig(
local_count=10 if scenario == "local_fit" else 2_000, counts=(10,) if scenario == "native_fit" else (2_000,)
)
target: Final = (
replace(_budget("responses"), model="anthropic/test") if scenario == "provider" else _budget("responses")
)
result: Final = await _run(
rig,
target=target,
payload={"input": [{"role": "user", "content": "latest"}]},
route="chat" if scenario == "chat" else "responses",
budgets=(),
)
assert (result is None) if scenario.endswith("fit") else isinstance(result, core.CompactionFailure)
assert rig.calls.empty()
assert rig.native.qsize() == (0 if scenario in ("local_fit", "chat", "provider") else 1)
@pytest.mark.parametrize(
("counts", "paid"),
(
*((((value,), 0)) for value in (-1, True, False, RuntimeError("private native response"))),
*((((2_000, value), 1)) for value in (-1, True, False, 2_000, RuntimeError("private native response"))),
),
)
async def test_native_count_failures_do_not_retry_paid_work(counts: tuple[int | Exception, ...], paid: int) -> None:
rig: Final = _Rig(counts=counts)
state: Final = core.CompactionState()
result: Final = await _run(rig, state=state)
if not isinstance(result, core.CompactionFailure):
pytest.fail("Invalid native count was accepted")
assert "private native response" not in result.message
assert isinstance(await _run(rig, state=state), core.CompactionFailure)
assert rig.calls.qsize() == paid
@pytest.mark.parametrize(
"change", ("same", "tail", "prefix", "instructions", "tools", "model", "protocol", "recipient")
)
async def test_memo_identity_and_one_call_per_request(change: str) -> None:
rig: Final = _Rig()
state: Final = core.CompactionState()
assert isinstance(await _run(rig, state=state), core.CompactedRequest)
changed: Final = {"role": "user", "content": "changed"}
patches: Final = {
"tail": {"input": [*_HISTORY[:-1], changed]},
"prefix": {"input": [changed, _HISTORY[-1]]},
"instructions": {"instructions": "changed"},
"tools": {"tools": [{"type": "function", "name": "lookup"}]},
}
if change == "model":
state.arm("different-compactors")
protocol: Final = "messages" if change == "protocol" else "responses"
target: Final = _budget(protocol, "retry" if change == "recipient" else "target")
result: Final = await _run(rig, protocol, state, {**_payload(protocol), **patches.get(change, {})}, target=target)
accepted: Final = change in ("same", "tail", "recipient")
assert isinstance(result, core.CompactedRequest if accepted else core.CompactionFailure)
assert rig.calls.qsize() == 1 and rig.native.qsize() == (4 if accepted else 3)
if isinstance(result, core.CompactedRequest):
assert result.value[-1] == (changed if change == "tail" else _HISTORY[-1])
assert isinstance(await _run(_Rig()), core.CompactedRequest)
@pytest.mark.parametrize("protocol", ("responses", "messages"))
@pytest.mark.parametrize(
"change", ("none", "id", "provider", "defaults", "oversized", "negative", "boolean", "allowance")
)
async def test_exact_selected_compactor_validation(protocol: NativeProtocol, change: str) -> None:
rig: Final = _Rig(counts=({"oversized": 100_000, "negative": -1, "boolean": True}.get(change, 100),))
target: Final = replace(_budget(protocol), input_limit=10_000, output_limit=core.SUMMARY_OUTPUT_TOKENS)
body: Final = {
**_payload(protocol),
**({"max_tokens": core.SUMMARY_OUTPUT_TOKENS} if protocol == "messages" else {}),
}
request: Final = core.NativeRequest(protocol, body, frozenset({target.deployment_id}))
selected: Final = replace(
target,
deployment_id="outsider" if change == "id" else target.deployment_id,
model=("anthropic/test" if protocol == "responses" else "openai/test")
if change == "provider"
else target.model,
)
overrides: Final = {"defaults": {"instructions": "overridden"}, "allowance": {"max_tokens": 1}}
payload: Final = {**body, **overrides.get(change, {})}
result: Final = await core.validate_native_recipient(request, selected, payload, 10, rig.native_counter)
accepted: Final = change == "none" or (change == "allowance" and protocol == "responses")
assert (result is None) if accepted else isinstance(result, core.CompactionFailure)
assert rig.native.qsize() == (1 if accepted or change in ("oversized", "negative", "boolean") else 0)
@pytest.mark.parametrize(
("patch", "reserve"),
(
*(
({key: value}, reserve)
for key in ("max_tokens", "max_completion_tokens", "max_output_tokens")
for value, reserve in ((None, 256), (64, 64), (0, None), (True, None), ("64", None), (257, None))
),
(dict[str, object](), 256),
({"max_tokens": 32, "max_output_tokens": 64}, 64),
({"thinking": {"budget_tokens": 64}}, 256),
({"max_tokens": 64, "thinking": {"budget_tokens": 64}}, None),
({"thinking": {"budget_tokens": True}}, None),
*(({"extra_body": body}, None) for body in (False, list[object](), "invalid")),
*(
({"extra_body": {key: "override"}}, None)
for key in ("input", "messages", "tools", "system", "instructions", "max_tokens", "thinking")
),
({"extra_body": {"provider_feature_enabled": True}}, 256),
),
)
async def test_output_reservation_and_extra_body_admission(patch: Mapping[str, object], reserve: int | None) -> None:
rig: Final = _Rig(local_count=1_750, counts=(1_750, 100))
result: Final = await _run(rig, payload={**_payload("responses"), **patch})
if reserve is None:
assert isinstance(result, core.CompactionFailure)
assert rig.local.empty() and rig.native.empty() and rig.calls.empty()
elif reserve == 64:
assert result is None and rig.calls.empty()
else:
assert isinstance(result, core.CompactedRequest) and rig.calls.qsize() == 1

View file

@ -13454,6 +13454,14 @@ def _tier_config(**overrides) -> Dict:
return {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, **overrides}
class TestContextWindowCompactionConfig:
@pytest.mark.parametrize("value", ["", " ", "\t\n", "\u2003", 17, {"model": "summary-model"}])
def test_rejects_blank_or_non_string_model_groups(self, value: object) -> None:
with pytest.raises(ValidationError, match="context_window_compaction_model"):
ComplexityRouterConfig.model_validate(
{"tiers": {"SIMPLE": "small-model"}, "context_window_compaction_model": value}
)
class TestContextWindowEscalation:
"""A tier decided on complexity alone must still hold the prompt, or the provider 400s.
@ -13621,6 +13629,38 @@ class TestContextWindowEscalation:
assert result.model == "small-model"
assert "context_escalated" not in result.routing_decision
@pytest.mark.asyncio
@pytest.mark.parametrize("compaction_model", [None, "summary-model"])
@pytest.mark.parametrize("session_affinity", [False, True])
async def test_compaction_keeps_the_selected_group_on_overflow(
self, compaction_model: str | None, session_affinity: bool
) -> None:
router: Final = ComplexityRouter(
model_name="test-router",
litellm_router_instance=_windowed_router(_SMALL, _BIG),
complexity_router_config={
"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"},
"session_affinity": session_affinity,
"context_window_compaction_model": compaction_model,
},
)
request_kwargs: Final = {"metadata": {"session_id": "compaction-session"}}
context_fit: Final = await router._request_context_fit(_OVERSIZED_TURNS, request_kwargs)
assert context_fit.accepts("small-model") is (compaction_model is not None)
original: Final = await router.async_pre_routing_hook(
model="test-router", request_kwargs=request_kwargs, messages=[{"role": "user", "content": "ok continue"}]
)
oversized: Final = await router.async_pre_routing_hook(
model="test-router", request_kwargs=request_kwargs, messages=_OVERSIZED_TURNS
)
assert original is not None and original.model == "small-model"
assert oversized is not None
assert oversized.model == (original.model if compaction_model else "big-model")
assert oversized.routing_decision.get("context_escalated", False) is (compaction_model is None)
if session_affinity:
assert oversized.routing_decision["cause"] == "session_affinity_pin"
@pytest.mark.asyncio
async def test_out_of_band_system_and_tools_count_against_the_window(self):
"""The Claude Code shape that live-testing caught: a tiny ask riding a top-level

View file

@ -1,4 +1,5 @@
from collections.abc import Mapping
from typing import Final
import pytest
@ -262,6 +263,39 @@ def test_strategy_router_dependencies(litellm_params, expected):
assert tuple((d.model_name, d.role) for d in found) == expected
@pytest.mark.parametrize("classifier_type", ["heuristic", "llm"])
@pytest.mark.parametrize("escalation_enabled", [False, True])
@pytest.mark.parametrize("compaction_model", [None, "summary-model", "tier-model"])
def test_compaction_dependency_is_independent_of_classification_and_escalation(
classifier_type: str, escalation_enabled: bool, compaction_model: str | None
) -> None:
found: Final = strategy_router_dependencies(
{
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {"SIMPLE": "tier-model"},
"classifier_type": classifier_type,
"classifier_llm_config": {"model": "classifier-model"},
"enable_context_window_escalation": escalation_enabled,
"context_window_compaction_model": compaction_model,
},
}
)
assert tuple(dependency.model_name for dependency in found if dependency.role == "compaction") == (
(compaction_model,) if compaction_model else ()
)
assert tuple(dependency.model_name for dependency in found if dependency.role == "tier") == ("tier-model",)
@pytest.mark.parametrize("value", ["", " ", "\t\n", 17, {"model": "summary-model"}])
def test_config_write_rejects_invalid_compaction_model(value: object) -> None:
violation: Final = validate_complexity_router_config_write(
complexity_router_config={"tiers": {"SIMPLE": "tier-model"}, "context_window_compaction_model": value}
)
assert violation is not None
assert "context_window_compaction_model" in violation
def test_complexity_default_model_param_wins_over_the_config_field():
"""ComplexityRouter overwrites config.default_model with the litellm_params one, so the
config field is dead whenever the param is set and must not be able to red the router."""
@ -333,6 +367,7 @@ def test_complexity_embedding_model_is_a_dependency_only_when_semantic_matching_
("tier_boundaries",),
("token_thresholds", "dimension_weights"),
("reasoning_override_min_score",),
("context_window_compaction_model",),
("tiers",),
],
)

View file

@ -42,6 +42,7 @@ from litellm.router import (
_anthropic_stream_raised_error_status,
_anthropic_stream_should_decline_fallback,
_anthropic_stream_should_drop_pre_content_ping,
_compaction_deployment_id,
_is_retriable_anthropic_status,
)
from litellm.router_strategy import simple_shuffle
@ -49,6 +50,374 @@ from litellm.types.llms.openai import ChatCompletionRequest
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy
from collections.abc import AsyncIterator, Sequence
from types import MappingProxyType
import pytest_asyncio
from pydantic import TypeAdapter
from litellm.router_strategy.complexity_router.context_compaction import ModelBudget, NativeRequest, native_request_scope
from litellm.types.llms.openai import AllMessageValues
_NATIVE_MODELS: Final = {"responses": "openai/gpt-5.6-luna", "messages": "anthropic/claude-sonnet-5"}
_NATIVE_PREFIX: Final = (
{"role": "user", "content": "historical record " * 5000},
{"role": "assistant", "content": "Recorded"},
)
_NATIVE_CANONICAL: Final = (
{"role": "user", "content": "Provider-retained canonical message"},
{"type": "compaction", "id": "cmp_native", "encrypted_content": "opaque-native-state"},
)
_NATIVE_BLOCK: Final = {"type": "compaction", "content": "Native state", "signature": "signed-native-state"}
_NATIVE_JSON: Final = TypeAdapter(dict[str, object])
@pytest_asyncio.fixture(loop_scope="function")
async def overflow_io(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[respx.MockRouter]:
from litellm import anthropic_beta_headers_manager
from litellm.litellm_core_utils import logging_worker
worker: Final = logging_worker.LoggingWorker(timeout=0.5)
monkeypatch.setattr(
anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG",
anthropic_beta_headers_manager.GetAnthropicBetaHeadersConfig.load_local_beta_headers_config(),
)
monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", worker)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
for provider in ("openai", "anthropic"):
monkeypatch.delenv(f"{provider.upper()}_API_KEY", raising=False)
monkeypatch.setattr(litellm, f"{provider}_key", None)
monkeypatch.setattr(litellm, "api_key", None)
try:
with respx.mock(assert_all_called=False) as transport:
transport.route(path__regex=r".*/(input_tokens|count_tokens)$").mock(side_effect=_native_wire_response)
transport.route(
host="native-compact.invalid", path__regex=r"/v1/(responses/compact|messages)$"
).mock(side_effect=_native_wire_response)
yield transport
await asyncio.wait_for(worker.flush(), timeout=5)
finally:
await worker.stop()
def _native_history(protocol: str) -> list[Mapping[str, object]]:
tail: Final = (
(
{"type": "function_call", "call_id": "call_lookup", "name": "lookup", "arguments": "{}"},
{"type": "function_call_output", "call_id": "call_lookup", "output": "MARIGOLD-482"},
)
if protocol == "responses"
else (
{"role": "assistant", "content": [{"type": "tool_use", "id": "lookup", "name": "lookup", "input": {}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "lookup", "content": "MARIGOLD-482"}]},
)
)
return [*_NATIVE_PREFIX, {"role": "user", "content": "What is the launch code?"}, *tail]
def _native_router(
protocol: str, *, scenario: str = "normal", pre_checks: bool = True, backup_limit: int = 64000
) -> Router:
wire_model: Final = _NATIVE_MODELS[protocol]
deployments: Final = (
("target", "a", 4096, wire_model), ("target", "b", 4096, wire_model),
("backup", "backup", backup_limit, wire_model),
("compact", "unfit" if scenario == "count" else "compact", 32000, wire_model),
("compact", "small", 4096, wire_model),
("compact", "foreign", 64000, _NATIVE_MODELS["messages" if protocol == "responses" else "responses"]),
("compact", "denied", 0, wire_model),
)
return Router(
model_list=[
{
"model_name": "native-auto",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {tier: "native-target" for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")},
"context_window_compaction_model": "native-compact",
"max_tokens_from_tier_model": False,
"deployment_affinity": True,
},
},
},
*(
{
"model_name": f"native-{group}",
"litellm_params": {
"model": model,
"api_base": f"https://native-{host}.invalid" + ("/v1" if model.startswith("openai/") else ""),
"api_key": "" if scenario == "credentials" and host == "compact" else f"test-{host}",
"region_name": "us" if scenario == "region" and group == "compact" else "eu",
**({"max_tokens": 32000} if scenario == "max-default" and host == "compact" else {}),
**(
{"instructions" if protocol == "responses" else "system": "Injected default"}
if scenario == "prompt-default" and host == "compact" else {}
),
},
"model_info": {
"id": host, "max_input_tokens": limit, "max_output_tokens": 8192,
"access_groups": (
["denied" if host == "denied" else "compact-access"] if group == "compact" else []
),
},
}
for group, host, limit, model in deployments
),
],
num_retries=0,
enable_pre_call_checks=pre_checks,
fallbacks=[{"native-target": ["native-backup"]}],
)
def _native_wire_response(request: httpx.Request) -> httpx.Response:
body: Final = _NATIVE_JSON.validate_json(request.content)
host: Final = request.url.host.removeprefix("native-").removesuffix(".invalid")
protocol: Final = "messages" if "/messages" in request.url.path else "responses"
credential: Final = request.headers.get("x-api-key") or request.headers.get("authorization", "").removeprefix(
"Bearer "
)
assert credential == f"test-{host}"
if request.url.path.endswith(("/input_tokens", "/count_tokens")):
tokens: Final = 40000 if host == "unfit" else 12000 if "historical record" in json.dumps(body) else 128
return httpx.Response(200, json={"input_tokens": tokens})
compact: Final = request.url.path.endswith("/compact") or body.get("compaction") == {"type": "summarize"}
if host == "compact":
assert compact, tuple(body)
if protocol == "responses" and compact:
assert not set(body).difference(
{"model", "input", "instructions", "previous_response_id", "prompt_cache_key", "prompt_cache_retention"}
)
response: Final = (
{
"id": "cmp_native" if compact else "resp_native", "created_at": 1, "model": body["model"],
"object": "response.compaction" if compact else "response",
"output": list(_NATIVE_CANONICAL) if compact else [{
"id": "msg_native", "type": "message", "role": "assistant", "status": "completed",
"content": [{"type": "output_text", "text": "MARIGOLD-482", "annotations": []}],
}],
"usage": {
"input_tokens": 12000 if compact else 128, "output_tokens": 8, "total_tokens": 12008 if compact else 136
},
}
if protocol == "responses" else {
"id": "msg_native", "type": "message", "role": "assistant", "model": body["model"],
"content": [_NATIVE_BLOCK] if compact else [{"type": "text", "text": "MARIGOLD-482"}],
"stop_reason": "compaction" if compact else "end_turn",
"usage": {
"input_tokens": 0, "output_tokens": 0,
"iterations": [{"type": "compaction", "input_tokens": 12000, "output_tokens": 8}],
} if compact else {"input_tokens": 128, "output_tokens": 8},
}
)
events: Final = (
(
{"type": "response.created", "sequence_number": 0, "response": {**response, "output": []}},
{"type": "response.output_text.delta", "sequence_number": 1, "item_id": "msg_native",
"output_index": 0, "content_index": 0, "delta": "MARIGOLD-482"},
{"type": "response.completed", "sequence_number": 2, "response": response},
)
if protocol == "responses" else (
{"type": "message_start", "message": {**response, "content": [], "stop_reason": None}},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "MARIGOLD-482"}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 8}},
{"type": "message_stop"},
)
)
if not body.get("stream"):
return httpx.Response(200, json=response)
return httpx.Response(
200, headers={"content-type": "text/event-stream"},
content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events),
)
async def _native_call(
router: Router,
protocol: str,
history: Sequence[Mapping[str, object]],
params: Mapping[str, object] = MappingProxyType({}),
) -> object:
kwargs: Final = {
"metadata": {"session_id": "native-pin", "user_api_key_auth": UserAPIKeyAuth(team_models=["compact-access"])},
**params,
}
if protocol == "responses":
return await router.aresponses(model="native-auto", input=list(history), max_output_tokens=128, **kwargs)
if protocol == "chat":
messages: Final = TypeAdapter(list[AllMessageValues]).validate_python(history)
return await router.acompletion(model="native-auto", messages=messages, max_tokens=128, **kwargs)
return await router.aanthropic_messages(model="native-auto", messages=list(history), max_tokens=128, **kwargs)
@pytest.mark.asyncio
async def test_autorouter_native_unarmed_helpers() -> None:
router: Final = Router(model_list=[])
deployment: Final[DeploymentTypedDict] = {
"model_name": "target", "litellm_params": {"model": "openai/gpt-5.6-luna"},
"model_info": {"id": "selected", "max_input_tokens": 4096, "max_output_tokens": 128},
}
assert _compaction_deployment_id(deployment) == "selected"
assert _compaction_deployment_id({}) == ""
assert _compaction_deployment_id({"model_info": {}}) == ""
assert _compaction_deployment_id({"model_info": {"id": 123}}) == ""
budget: Final = router._compaction_model_budget(deployment)
assert isinstance(budget, ModelBudget) and budget.deployment_id == "selected"
payload: Final = router._compaction_request_payload(deployment, {"input": "Hello"})
assert payload["input"] == "Hello" and await router._acount_compaction_tokens(budget.model, payload) > 0
assert await router._aprepare_selected_deployment(
deployment, "target", {"_context_window_compaction_state": {"model": "forged"}}
) is None
selected: Final = Router(model_list=[deployment])
native: Final = NativeRequest("responses", {}, frozenset({"unavailable"}))
with native_request_scope(native), pytest.raises(litellm.ContextWindowExceededError, match="No eligible"):
selected._common_checks_available_deployment("target", request_kwargs={})
legacy: Final = Router(model_list=_native_router("responses").model_list, routing_strategy="usage-based-routing")
with pytest.raises(litellm.ContextWindowExceededError, match="legacy synchronous deployment selection"):
await legacy.async_get_available_deployment(model="native-auto", request_kwargs={})
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ("responses", "messages"))
@pytest.mark.parametrize("stream", (False, True))
@pytest.mark.parametrize("pre_checks", (False, True))
async def test_autorouter_native_pinned_wire(
protocol: str, stream: bool, pre_checks: bool, overflow_io: respx.MockRouter
) -> None:
router: Final = _native_router(protocol, pre_checks=pre_checks, scenario="max-default" if pre_checks else "normal")
history: Final = _native_history(protocol)
original: Final = copy.deepcopy(history)
parameters: Final = {"type": "object", "properties": {}}
context: Final = {
"instructions" if protocol == "responses" else "system": "Retain request instructions",
"tools": [
{"type": "function", "name": "lookup", "parameters": parameters}
if protocol == "responses" else {"name": "lookup", "input_schema": parameters}
],
}
targets: Final = overflow_io.route(
host__regex=r"native-[ab]\.invalid", path__regex=r"/v1/(responses|messages)$"
).mock(side_effect=_native_wire_response)
for request_history in ([{"role": "user", "content": "Hello"}], history):
result: Final = await _native_call(
router, protocol, request_history, {**context, "stream": stream, "disable_fallbacks": True}
)
if isinstance(result, AsyncIterator):
assert [chunk async for chunk in result]
assert sum(call.request.url.host == "native-compact.invalid" for call in overflow_io.calls) == (
2 if request_history is history else 0
)
assert targets.call_count == 2 and targets.calls[0].request.url.host == targets.calls[1].request.url.host
field: Final = "input" if protocol == "responses" else "messages"
canonical: Final = (
list(_NATIVE_CANONICAL) if protocol == "responses" else [{"role": "assistant", "content": [_NATIVE_BLOCK]}]
)
sent: Final = _NATIVE_JSON.validate_json(targets.calls[-1].request.content)
assert sent[field] == [*canonical, *history[2:]] and history == original
assert all(sent[key] == value for key, value in context.items())
assert "_context_window_compaction_state" not in sent
calls: Final = tuple(call.request for call in overflow_io.calls)
compact: Final = next(
request for request in calls if request.url.host == "native-compact.invalid"
and not request.url.path.endswith(("/input_tokens", "/count_tokens"))
)
compact_payload: Final = _NATIVE_JSON.validate_json(compact.content)
assert compact_payload[field] == history[:2]
assert compact_payload.get("tools") == (context["tools"] if protocol == "messages" else None)
assert {request.url.host for request in calls} == {targets.calls[0].request.url.host, "native-compact.invalid"}
counted: Final = [request for request in calls if request.url.path.endswith(("/input_tokens", "/count_tokens"))]
assert len(counted) == 3 and _NATIVE_JSON.validate_json(counted[-1].content)[field] == sent[field]
assert all(_NATIVE_JSON.validate_json(counted[-1].content)[key] == sent[key] for key in context)
if protocol == "messages":
assert compact_payload["max_tokens"] == 4096
assert compact_payload["compaction"] == {"type": "summarize"}
assert compact_payload["stream"] is False and compact_payload["system"] == context["system"]
for request in (*counted, compact, targets.calls[-1].request):
assert "compact-2026-09-04" in request.headers.get("anthropic-beta", ""), str(request.url)
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ("responses", "messages"))
@pytest.mark.parametrize("stream", (False, True))
@pytest.mark.parametrize("backup_limit", (4096, 64000))
async def test_autorouter_native_retry_fallback_original_history(
protocol: str, stream: bool, backup_limit: int, overflow_io: respx.MockRouter
) -> None:
error: Final = {
"type": "error", "sequence_number": 0,
"error": {"type": "server_error" if protocol == "responses" else "overloaded_error" if stream else "api_error",
"code": "server_error", "message": "retry"},
}
targets: Final = overflow_io.route(
host__regex=r"native-[ab]\.invalid", path__regex=r"/v1/(responses|messages)$"
).respond(
200 if stream else 503 if protocol == "responses" else 500,
headers={"content-type": "text/event-stream" if stream else "application/json"},
content=f"event: error\ndata: {json.dumps(error)}\n\n" if stream else json.dumps(error),
)
backup: Final = overflow_io.route(host="native-backup.invalid", path__regex=r"/v1/(responses|messages)$").mock(
side_effect=_native_wire_response
)
history: Final = _native_history(protocol)
result: Final = await _native_call(
_native_router(protocol, backup_limit=backup_limit), protocol, history,
{"stream": stream, "num_retries": int(not stream), "max_retries": 0,
"model_group_retry_policy": {
"native-auto": RetryPolicy(ServiceUnavailableErrorRetries=1, InternalServerErrorRetries=1)
}},
)
if isinstance(result, AsyncIterator):
assert [chunk async for chunk in result]
field: Final = "input" if protocol == "responses" else "messages"
compacted: Final = _NATIVE_JSON.validate_json(targets.calls[0].request.content)[field]
assert targets.call_count == (1 if stream else 2) and backup.call_count == 1
assert _NATIVE_JSON.validate_json(targets.calls[-1].request.content)[field] == compacted
assert _NATIVE_JSON.validate_json(backup.calls[0].request.content)[field] == (
history if backup_limit == 64000 else compacted
)
assert sum(call.request.url.host == "native-compact.invalid" for call in overflow_io.calls) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ("responses", "messages"))
@pytest.mark.parametrize("reason", ("credentials", "region", "prompt-default", "count", "access"))
async def test_autorouter_native_compactor_admission(
protocol: str, reason: str, overflow_io: respx.MockRouter
) -> None:
router: Final = _native_router(protocol, scenario=reason)
params: Final = {
"disable_fallbacks": True,
**({"allowed_model_region": "eu"} if reason == "region" else {}),
**(
{"metadata": {"user_api_key_auth": UserAPIKeyAuth(team_models=["unrelated"])}}
if reason == "access" else {}
),
}
with pytest.raises(litellm.ContextWindowExceededError):
await _native_call(router, protocol, _native_history(protocol), params)
assert all(call.request.url.path.endswith(("/input_tokens", "/count_tokens")) for call in overflow_io.calls)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"protocol,provider", (("chat", "responses"), ("responses", "messages"), ("messages", "responses"))
)
async def test_autorouter_native_unsupported_overflow(
protocol: str, provider: str, overflow_io: respx.MockRouter
) -> None:
history: Final = [*_NATIVE_PREFIX, {"role": "user", "content": "Hello"}]
with pytest.raises(litellm.ContextWindowExceededError, match="unsupported"):
await _native_call(_native_router(provider), protocol, history, {"disable_fallbacks": True})
assert not overflow_io.calls
def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
# initialize a real Router (env‑vars can be empty)
router = litellm.Router(

View file

@ -35905,6 +35905,11 @@ export interface components {
* @description Keywords indicating code-related content
*/
code_keywords?: string[] | null;
/**
* Context Window Compaction Model
* @description Ordinary Router model group used for provider-native compaction when an asynchronous Responses/OpenAI or Messages/Anthropic request exceeds its selected deployment's input budget. The compactor must fit the input and its native output must fit the unchanged answering deployment. Unsupported overflow routes fail explicitly. Compaction takes precedence over context window escalation and incurs a separate billable call under normal access and limits. Nested Auto Routers are not supported. Omit or set null to keep existing routing behavior.
*/
context_window_compaction_model?: string | null;
/**
* Context Window Escalation Buffer
* @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens.