feat(router): native compact-to-fit across conversation APIs

This commit is contained in:
Tin Chi Lo 2026-09-19 19:03:06 -07:00
parent daecea3eb8
commit d1a74f0214
33 changed files with 1545 additions and 47 deletions

View file

@ -7,6 +7,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",

View file

@ -1,10 +1,24 @@
import re
from collections.abc import Iterator, Mapping
from collections.abc import Generator, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any, Final
from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
_CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
"inherited_message_logging_disabled", default=False
)
@contextmanager
def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled)
try:
yield
finally:
_inherited_message_logging_disabled.reset(token)
def iter_client_callback_metadata_dicts(
@ -143,7 +157,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple
def initialize_standard_callback_dynamic_params(
kwargs: dict | None = None,
kwargs: dict[str, object] | None = None,
) -> StandardCallbackDynamicParams:
"""
Initialize the standard callback dynamic params from the kwargs
@ -179,4 +193,10 @@ def initialize_standard_callback_dynamic_params(
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
if _inherited_message_logging_disabled.get():
private_params: Final[StandardCallbackDynamicParams] = {
**standard_callback_dynamic_params,
"turn_off_message_logging": True,
}
return private_params
return standard_callback_dynamic_params

View file

@ -96,6 +96,7 @@ from ..common_utils import (
AnthropicModelInfo,
eager_input_streaming_flag,
process_anthropic_headers,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
)
@ -1771,7 +1772,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return tools
def _ensure_beta_header(self, headers: dict, beta_value: str) -> None:
def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None:
"""
Ensure a beta header value is present in the anthropic-beta header.
Merges with existing values instead of overriding them.
@ -1780,13 +1781,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers: Dictionary of headers to update
beta_value: The beta header value to add
"""
existing_beta: Final = headers.get("anthropic-beta")
if existing_beta is None:
headers["anthropic-beta"] = beta_value
return
existing_values: Final = [beta.strip() for beta in existing_beta.split(",")]
if beta_value not in existing_values:
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
existing_values: Final = tuple(
beta.strip()
for key, value in headers.items()
if key.lower() == "anthropic-beta"
for beta in value.split(",")
if beta.strip()
)
for key in tuple(headers):
if key.lower() == "anthropic-beta":
headers.pop(key)
headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value)))
def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None:
"""
@ -1824,7 +1829,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
)
def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict:
def update_headers_with_optional_anthropic_beta(
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
) -> dict:
"""Update headers with optional anthropic beta."""
# Skip adding beta headers for Vertex requests
@ -1833,6 +1840,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if is_vertex_request:
return headers
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
_tools: Final = optional_params.get("tools", [])
for tool in _tools:
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
@ -1929,8 +1939,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
custom_llm_provider=self._resolved_provider,
)
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
@ -1977,6 +1985,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
message=f"{e}\nReceived Messages={messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(
headers=headers, optional_params=optional_params, messages=anthropic_messages
)
## Auto-strip advisor blocks from history if advisor tool is absent.
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
_all_tools: Final = optional_params.get("tools") or []

View file

@ -77,6 +77,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
)
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching

View file

@ -0,0 +1,92 @@
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import TypeAdapter
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction
from litellm.types.llms.openai import AllMessageValues
if TYPE_CHECKING:
from litellm.router import Router
Protocol: TypeAlias = Literal["chat", "messages"]
_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_MESSAGES: Final = TypeAdapter(list[AllMessageValues])
_HEADERS: Final = TypeAdapter(dict[str, str])
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice")
def supports_native_compaction(params: Mapping[str, object]) -> bool:
from litellm.utils import get_model_info
if params.get("custom_llm_provider") not in (None, "anthropic", "openai"):
return False
model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/")
try:
return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True
except Exception:
return False
def compatible_defaults(payload: Mapping[str, object]) -> bool:
return all(payload.get(key) is None for key in _CONFLICTS)
def request_kwargs() -> Mapping[str, object]:
operation: Final[AnthropicCompaction] = {"type": "summarize"}
return MappingProxyType(
{
"compaction": operation,
"extra_headers": _HEADERS.validate_python(
MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
),
}
)
def _native_blocks(protocol: Protocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
if protocol == "messages":
return (
_OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else ()
)
choices: Final = _OBJECTS.validate_python(response.get("choices", ()))
choice: Final = choices[0] if len(choices) == 1 else _EMPTY
message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY))
fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY)
return _OBJECTS.validate_python(fields.get("compaction_blocks", ()))
def extract_summary(protocol: Protocol, response: Mapping[str, object]) -> str | None:
blocks: Final = _native_blocks(protocol, response)
block: Final = blocks[0] if len(blocks) == 1 else _EMPTY
content: Final = block.get("content")
return (
content
if block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and block.get("signature")
and isinstance(content, str)
and content.strip()
else None
)
async def dispatch(router: Router, protocol: Protocol, payload: Mapping[str, object]) -> Mapping[str, object]:
if protocol == "messages":
return _MAPPING.validate_python(
await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
)
response: Final = await router.acompletion(
model=str(payload["model"]),
messages=_MESSAGES.validate_python(payload["messages"]),
stream=False,
**MappingProxyType(
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
),
)
return _MAPPING.validate_python(response.model_dump())

View file

@ -4,6 +4,8 @@ import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
@ -16,6 +18,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
def _optional_attr(source: object, name: str) -> object:
@ -36,6 +39,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
return signature if isinstance(signature, str) else None
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
fields: Final = _as_string_mapping(provider_specific_fields)
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
return (
tuple(
block
for raw_block in raw_blocks
if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction"
)
if isinstance(raw_blocks, (list, tuple))
else ()
)
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
{"name", "type", "input_schema", "description", "cache_control", "strict"}
)
@ -1330,7 +1347,11 @@ class LiteLLMAnthropicMessagesAdapter:
tool_name_mapping: dict[str, str] | None = None,
) -> list[dict[str, Any]]:
new_content: Final[list[dict[str, Any]]] = []
for choice in choices:
for choice, compaction_blocks in (
(choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))
for choice in choices
):
new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks)
# Handle thinking blocks first
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
for thinking_block in choice.message.thinking_blocks:
@ -1365,7 +1386,7 @@ class LiteLLMAnthropicMessagesAdapter:
)
# Handle text content
if choice.message.content is not None:
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
new_content.append(
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
)
@ -1545,21 +1566,31 @@ class LiteLLMAnthropicMessagesAdapter:
openai_finish_reason=openai_finish_reason
)
anthropic_finish_reason: Final = (
"refusal"
"compaction"
if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction"
else "refusal"
if refusal_text is not None and translated_finish_reason != "max_tokens"
else translated_finish_reason
)
# extract usage
usage: Final[Usage] = getattr(response, "usage")
anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
if polyfill_result is not None and polyfill_result.iterations_usage is not None:
message_iteration: Final[UsageIteration] = {
"type": "message",
"input_tokens": anthropic_usage["input_tokens"],
"output_tokens": usage.completion_tokens or 0,
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
anthropic_usage: Final[AnthropicUsage] = (
{
**message_usage,
"iterations": [
*polyfill_iterations,
UsageIteration(
type="message",
input_tokens=message_usage.get("input_tokens", 0),
output_tokens=usage.completion_tokens or 0,
),
],
}
anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration]
if polyfill_iterations is not None
else message_usage
)
translated_obj: Final = AnthropicMessagesResponse(
id=response.id,

View file

@ -580,7 +580,9 @@ def anthropic_messages_handler(
)
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,

View file

@ -24,6 +24,7 @@ from ...common_utils import (
AnthropicError,
AnthropicModelInfo,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
"output_format",
"inference_geo",
"speed",
@ -633,6 +635,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
beta_values.update(existing_beta)
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:

View file

@ -14487,6 +14487,7 @@
"source": "https://docs.anthropic.com/en/docs/about-claude/pricing"
},
"claude-sonnet-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
@ -14526,6 +14527,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-sonnet-4-6": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -14741,6 +14743,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-6": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14779,6 +14782,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-6-20260205": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14816,6 +14820,7 @@
"prompt_cache_min_tokens": 4096
},
"claude-opus-4-7": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14855,6 +14860,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-7-20260416": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14893,6 +14899,7 @@
"prompt_cache_min_tokens": 2048
},
"claude-fable-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -14932,6 +14939,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-fable-5-1": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
@ -14972,6 +14980,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -15014,6 +15023,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-8": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -60334,6 +60344,7 @@
"supports_audio_output": true
},
"claude-mythos-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -60373,6 +60384,7 @@
}
},
"claude-mythos-5-1": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
@ -60413,6 +60425,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-mythos-preview": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,

View file

@ -93,6 +93,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
)
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails
from litellm.router import Router
@ -2533,7 +2534,7 @@ class ProxyBaseLLMRequestProcessing:
user_model=user_model,
user_api_key_dict=user_api_key_dict,
)
llm_call_task: Final = asyncio.create_task(llm_call)
llm_call_task: Final = asyncio.create_task(with_proxy_compaction_executor(llm_call, request))
tasks.append(llm_call_task)
llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call

View file

@ -0,0 +1,91 @@
import asyncio
from collections.abc import Awaitable, Mapping
from contextvars import Context
from types import MappingProxyType
from typing import Final, Literal, TypeVar
import httpx
from fastapi import Request
from pydantic import TypeAdapter, ValidationError
from starlette.types import ASGIApp
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
inherit_message_logging_privacy,
initialize_standard_callback_dynamic_params,
)
from litellm.proxy.litellm_pre_call_utils import UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS
from litellm.router_strategy.complexity_router.context_compaction import (
compaction_executor,
native_compaction_call,
)
_ResultT: Final = TypeVar("_ResultT")
_ASGI_APP: Final = TypeAdapter[ASGIApp](ASGIApp)
_ROOT_PATH: Final = TypeAdapter(str)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_REMOVED_HEADERS: Final = frozenset(
(
b"content-length",
b"x-litellm-call-id",
b"x-litellm-num-retries",
b"x-litellm-timeout",
b"x-litellm-stream-timeout",
)
)
async def with_proxy_compaction_executor(call: Awaitable[_ResultT], request: Request) -> _ResultT:
async def execute(protocol: Literal["chat", "messages"], payload: Mapping[str, object]) -> Mapping[str, object]:
logging_disabled: Final = initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is True
async def dispatch() -> Mapping[str, object]:
scope: Final = _JSON_OBJECT.validate_python(request.scope)
root_path: Final = _ROOT_PATH.validate_python(scope.get("root_path", ""))
path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages"
url: Final = str(request.url.replace(path=root_path.rstrip("/") + path, query="", fragment=""))
headers: Final = tuple(
(name, value)
for name, value in request.headers.raw
if name.lower() not in _REMOVED_HEADERS
and not (logging_disabled and name.decode("latin-1").lower() in UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS)
)
transport: Final = httpx.ASGITransport(
app=_ASGI_APP.validate_python(scope["app"]),
root_path=root_path,
client=request.client, # pyright: ignore[reportArgumentType] # ASGI permits a missing client address
raise_app_exceptions=False,
)
with native_compaction_call(), inherit_message_logging_privacy(logging_disabled):
async with httpx.AsyncClient(
transport=transport,
follow_redirects=False,
timeout=httpx.Timeout(None),
) as client:
async with client.stream(
"POST", url, headers=headers, json=_JSON_OBJECT.validate_python(payload)
) as response:
if not response.is_success:
raise BadRequestError(
message=f"Native compaction child request failed (HTTP {response.status_code})",
model="context_compaction",
llm_provider="",
)
body: Final = await response.aread()
try:
return MappingProxyType(_JSON_OBJECT.validate_json(body))
except ValidationError:
raise BadRequestError(
message="Native compaction child returned an invalid JSON object",
model="context_compaction",
llm_provider="",
) from None
task: Final = Context().run(asyncio.create_task, dispatch())
return await task
token: Final = compaction_executor.set(execute)
try:
return await call
finally:
compaction_executor.reset(token)

View file

@ -118,6 +118,15 @@ from litellm.llms.openai_like.model_info import (
get_openai_compatible_model_info,
)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.complexity_router.context_compaction import (
arm_compaction,
compact_to_fit,
compaction_pending,
initialize_compaction_state,
is_native_compaction_call,
reject_recursive_compactor,
surface_for_call,
)
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
@ -3636,6 +3645,7 @@ class Router:
kwargs=kwargs,
client_type="max_parallel_requests",
)
compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat")
async with contextlib.AsyncExitStack() as deployment_slot:
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
deployment_slot.enter_context(max_parallel_requests_limit)
@ -3644,7 +3654,7 @@ class Router:
logging_obj=logging_obj,
parent_otel_span=parent_otel_span,
)
response = await litellm.acompletion(**input_kwargs)
response = await litellm.acompletion(**compacted_input)
## CHECK CONTENT FILTER ERROR ##
if isinstance(response, ModelResponse):
@ -5242,8 +5252,14 @@ class Router:
if custom_llm_provider is not None:
response_kwargs["custom_llm_provider"] = custom_llm_provider
compacted_input: Final = await compact_to_fit(
self,
deployment,
response_kwargs,
surface_for_call(getattr(original_generic_function, "__name__", "")),
)
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
response = await original_generic_function(**response_kwargs)
response = await original_generic_function(**compacted_input)
if self._should_raise_anthropic_refusal_error(
model=model,
@ -7401,6 +7417,13 @@ class Router:
If it fails after num_retries, fall back to another model group
"""
model_group: Final[str | None] = kwargs.get("model")
if (
surface_for_call(
getattr(kwargs.get("original_generic_function") or kwargs.get("original_function"), "__name__", "")
)
is not None
):
kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs)
clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary
if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets):
_fallback_metadata_key: Final = _get_router_metadata_variable_name(
@ -12109,8 +12132,8 @@ class Router:
def _count_pre_call_check_tokens(
self,
messages: list[dict[str, str]] | None,
input: str | list | None,
messages: Sequence[Mapping[str, object]] | None,
input: str | list[object] | None,
request_kwargs: Mapping[str, object] | None = None,
) -> int:
"""
@ -12257,7 +12280,9 @@ class Router:
_rate_limit_error = False
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs)
has_countable_input: Final = messages is not None or input is not None
has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending(
request_kwargs
)
## get model group RPM ##
dt: Final = get_utc_datetime()
@ -13456,6 +13481,8 @@ class Router:
registered_model_name: str,
request_kwargs: Mapping[str, object],
) -> str:
if is_native_compaction_call():
return registered_model_name
if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)):
return registered_model_name
cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs)
@ -13534,6 +13561,7 @@ class Router:
model=registered_model_name, request_kwargs=request_kwargs
)
if selected_strategy is None:
arm_compaction(request_kwargs, None)
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
self._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
@ -13544,6 +13572,24 @@ class Router:
return None
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
reject_recursive_compactor(registered_model_name)
arm_compaction(
request_kwargs,
selected_strategy.strategy.config.context_compaction
if isinstance(selected_strategy.strategy, ComplexityRouter)
else None,
tuple(
dict.fromkeys(
member
for pool in selected_strategy.strategy.config.tiers.values()
for member in ((pool,) if isinstance(pool, str) else pool)
)
)
if isinstance(selected_strategy.strategy, ComplexityRouter)
else (),
)
await authorize_member_auto_router_inference(
deployment=self._selected_strategy_marker_deployment(

View file

@ -400,6 +400,31 @@ affinity write happens upstream of the gate and stores the session's own model,
turn replays the original pin and the override is never pinned in its place. It does nothing
unless `modality_routing` is also on.
### Native compact-to-fit
The complexity router keeps the selected answering deployment when full conversation history approaches its input limit. This is enabled by default for async `/v1/chat/completions`, `/v1/responses` and `/v1/messages`. It takes precedence over context-window escalation for those requests
The router selects a native-capable compactor from its configured tier groups, checks that the older history fits, and requests Anthropic's on-demand native compaction. It inserts the returned summary as ordinary assistant text, preserving instructions, tools and the latest user turn. The answering model can be Haiku, DeepSeek, Kimi, GPT or another configured model; it does not need to support compaction itself. A larger context window alone does not make a model a native compactor
Native capability comes from `supports_anthropic_compaction` in model metadata. Selection skips groups containing an unsupported, undersized or incompatibly configured deployment. No unrelated gateway model is selected automatically. To name a separate compactor or change the defaults:
```yaml
complexity_router_config:
context_compaction:
model: native-compactor
trigger_ratio: 0.9
max_tokens: 4096
timeout_seconds: 120
```
Omit `context_compaction` to select automatically. Set it to `false` or `null` to disable it. The compactor must be a regular configured model group. The caller must have access to it and sufficient budget; configuring it grants no additional access
The input budget reserves the effective output allowance and a safety margin. Counts are estimates, so the provider remains authoritative. The router rechecks the compacted payload before sending it, attempts compaction only once per request, and reuses that attempt across answering retries. Compaction and answering are separate billed calls; output usage and therefore the final cost are not exactly knowable in advance
Responses compaction supports complete text history with ordinary function calls and results. Only its older prefix is converted for the compactor; retained items stay in their original format. Stored history (`previous_response_id` or `conversation`), explicit native context management, and opaque reasoning/compaction items remain on their existing client-managed path. This feature does not convert OpenAI encrypted state or claim it can be replayed across providers. The existing `/responses/compact` endpoint remains available for native OpenAI replay
A single oversized user turn, unclosed tool exchanges, unsupported Responses content, missing compactor capacity or an oversized summary produces a clear error instead of silently dropping context or changing the answerer. Fitting requests need no compaction. Models with unknown input limits retain their existing admission behavior
### Session pin retention
`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.

View file

@ -56,6 +56,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
from litellm.router_strategy.complexity_router.tier_predictor import (
TierSuccessPredictor,
resolve_tier_artifact,
@ -3257,7 +3258,11 @@ class ComplexityRouter(CustomLogger):
resolved_messages: Sequence[Mapping[str, object]] | None,
request_kwargs: Mapping[str, object],
) -> _RequestContextFit:
if not self.config.enable_context_window_escalation or not resolved_messages:
if (
compaction_pending(request_kwargs)
or not self.config.enable_context_window_escalation
or not resolved_messages
):
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
(self.config.default_model,) if self.config.default_model else ()
@ -3283,7 +3288,11 @@ class ComplexityRouter(CustomLogger):
(the placement stands). Only a real tokenizer count ever moves a request, escalation
lands only on groups whose every deployment declares a fitting window, and a group
with no resolvable window is never moved on faith in either direction."""
if not self.config.enable_context_window_escalation or not resolved_messages:
if (
compaction_pending(request_kwargs)
or not self.config.enable_context_window_escalation
or not resolved_messages
):
return None
pools: Final = self._tier_pools()
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))

View file

@ -833,6 +833,15 @@ class CustomDimension(BaseModel):
)
class ContextCompactionConfig(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
model: str | None = Field(default=None, min_length=1)
trigger_ratio: float = Field(default=0.9, gt=0, lt=1)
max_tokens: int = Field(default=4096, ge=512)
timeout_seconds: float = Field(default=120, gt=0)
class ComplexityRouterConfig(BaseModel):
"""Configuration for the ComplexityRouter."""
@ -1304,6 +1313,16 @@ class ComplexityRouterConfig(BaseModel):
),
)
context_compaction: ContextCompactionConfig | Literal[False] = Field(
default_factory=ContextCompactionConfig,
description="Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.",
)
@field_validator("context_compaction", mode="before")
@classmethod
def _normalize_context_compaction(cls, value: object) -> object:
return False if value is None else value
enable_context_window_escalation: bool = Field(
default=True,
description=(

View file

@ -0,0 +1,395 @@
from __future__ import annotations
import asyncio
import hashlib
import json
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from itertools import takewhile
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol, TypeAlias
from pydantic import TypeAdapter
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
inherit_message_logging_privacy,
initialize_standard_callback_dynamic_params,
)
from litellm.litellm_core_utils.internal_call_metadata import parent_session_kwargs, sanitized_forwardable_call_metadata
from litellm.litellm_core_utils.redact_messages import (
should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # legacy privacy owner accepts validated call details
)
from litellm.llms.anthropic import compaction as native
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
if TYPE_CHECKING:
from litellm.router import Router
Surface: TypeAlias = Literal["chat", "messages", "responses"]
_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType(
{"_acompletion": "chat", "anthropic_messages": "messages", "aresponses": "responses"}
)
_MAPPING: Final = TypeAdapter(Mapping[str, object])
_DICT: Final = TypeAdapter(dict[str, object])
_ITEMS: Final = TypeAdapter(list[dict[str, object]])
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_INPUT: Final = TypeAdapter[str | list[object] | None](str | list[object] | None)
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_STATE_KEY: Final = "_context_compaction_state"
_native_child: Final[ContextVar[bool]] = ContextVar("native_compaction_child", default=False)
class CompactionExecutor(Protocol):
async def __call__(self, protocol: native.Protocol, payload: Mapping[str, object]) -> Mapping[str, object]: ...
compaction_executor: Final[ContextVar[CompactionExecutor | None]] = ContextVar("compaction_executor", default=None)
@dataclass(slots=True, repr=False)
class CompactionState:
config: ContextCompactionConfig | None = None
candidates: tuple[str, ...] = ()
summary: tuple[str, asyncio.Task[str]] | None = None
def surface_for_call(function_name: str) -> Surface | None:
return _SURFACES.get(function_name)
@dataclass(frozen=True, slots=True)
class InputBudget:
window: int | None
available: int | None
@contextmanager
def native_compaction_call() -> Generator[None]:
token: Final = _native_child.set(True)
try:
yield
finally:
_native_child.reset(token)
def initialize_compaction_state(kwargs: Mapping[str, object]) -> CompactionState:
existing: Final = kwargs.get(_STATE_KEY)
return existing if isinstance(existing, CompactionState) else CompactionState()
def arm_compaction(
kwargs: Mapping[str, object],
config: ContextCompactionConfig | Literal[False] | None,
candidates: tuple[str, ...] = (),
) -> None:
state: Final = kwargs.get(_STATE_KEY)
if isinstance(state, CompactionState):
state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None
state.candidates = candidates
def _client_managed(payload: Mapping[str, object]) -> bool:
return any(
payload.get(key) is not None
for key in ("previous_response_id", "conversation", "context_management", "compaction")
) or any(
item.get("type") in ("reasoning", "compaction", "item_reference")
or item.get("encrypted_content") is not None
or any(block.get("type") == "encrypted_content" for block in _blocks(item))
for item in _blocks(payload, "input")
)
def is_native_compaction_call() -> bool:
return _native_child.get()
def reject_recursive_compactor(model: str) -> None:
if _native_child.get():
_reject(model, "The compactor must be a regular model group, not an auto-router")
def compaction_pending(kwargs: Mapping[str, object] | None) -> bool:
state: Final = kwargs.get(_STATE_KEY) if kwargs is not None else None
return isinstance(state, CompactionState) and state.config is not None
def _reject(model: str, reason: str) -> NoReturn:
from litellm.exceptions import BadRequestError
raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="")
def _blocks(item: Mapping[str, object], key: str = "content") -> tuple[Mapping[str, object], ...]:
value: Final = item.get(key)
return _OBJECTS.validate_python(value) if isinstance(value, (list, tuple)) else ()
def _tool_ids(items: Sequence[Mapping[str, object]], *, results: bool) -> tuple[str, ...]:
return tuple(
identifier if isinstance(identifier, str) else ""
for item in items
for identifier in (
*((item.get("tool_call_id"),) if results and item.get("role") == "tool" else ()),
*(
(item.get("call_id"),)
if item.get("type") == ("function_call_output" if results else "function_call")
else ()
),
*(
block.get("tool_use_id" if results else "id")
for block in _blocks(item)
if block.get("type") == ("tool_result" if results else "tool_use")
),
*(call.get("id") for call in _blocks(item, "tool_calls") if not results),
)
)
def _history(
items: Sequence[Mapping[str, object]], model: str
) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]:
instructions: Final = tuple(takewhile(lambda item: item.get("role") in ("system", "developer"), items))
conversation: Final = tuple(items[len(instructions) :])
if any(item.get("role") in ("system", "developer") for item in conversation):
_reject(model, "Mid-conversation instructions cannot be compacted")
split: Final = next(
(
index
for index in range(len(conversation) - 1, -1, -1)
if conversation[index].get("role") == "user"
and not any(block.get("type") == "tool_result" for block in _blocks(conversation[index]))
),
0,
)
prefix: Final = conversation[:split]
calls: Final = _tool_ids(prefix, results=False)
results: Final = _tool_ids(prefix, results=True)
if (
not prefix
or "" in calls
or "" in results
or len(calls) != len(frozenset(calls))
or sorted(calls) != sorted(results)
):
_reject(model, "No closed older conversation is available without changing the latest request")
return instructions, prefix, conversation[split:]
async def _count(router: Router, payload: Mapping[str, object]) -> int:
return await asyncio.to_thread(
router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter
messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None,
input=_INPUT.validate_python(payload.get("input")),
request_kwargs=payload,
)
def _budget(
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
) -> InputBudget:
model: Final = str(payload.get("model", ""))
info: Final = _MAPPING.validate_python(
router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
)
raw_window: Final = info.get("max_input_tokens")
window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None
output: Final = next(
(
payload[key]
for key in ("max_completion_tokens", "max_output_tokens", "max_tokens")
if payload.get(key) is not None
),
info.get("max_output_tokens"),
)
if output is not None and (not isinstance(output, int) or isinstance(output, bool) or output <= 0):
_reject(model, "The output allowance must be a positive integer")
return InputBudget(window, int(window * ratio) - output if window is not None and isinstance(output, int) else None)
async def _compactor_model(router: Router, state: CompactionState, payload: Mapping[str, object]) -> str:
needed: Final = await _count(router, payload)
candidates: Final = (
(state.config.model,) if state.config is not None and state.config.model is not None else state.candidates
)
selected: Final = next(
(
candidate
for candidate in candidates
if (deployments := tuple(router.get_model_list(model_name=candidate) or ()))
and all(
native.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"]))
and native.compatible_defaults(params)
and (budget := _budget(router, deployment, payload, 0.9)).available is not None
and needed <= budget.available
for deployment in deployments
)
),
None,
)
return (
selected
if selected is not None
else _reject(
str(payload["model"]),
"No configured compactor supports native compaction with enough context and compatible defaults",
)
)
def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[str, object]:
if surface != "responses":
return payload
from openai.types.responses.response_create_params import ResponseInputParam
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=TypeAdapter(ResponseInputParam).validate_python(payload["input"]),
responses_api_request=_DICT.validate_python(payload),
)
tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
_OBJECTS.validate_python(payload.get("tools") or ())
)
return MappingProxyType({**payload, "messages": _ITEMS.validate_python(messages), "tools": tools})
async def _generate_summary(
router: Router, protocol: native.Protocol, payload: Mapping[str, object], timeout: float
) -> str:
executor: Final = compaction_executor.get()
with native_compaction_call():
response: Final = await asyncio.wait_for(
executor(protocol, payload) if executor is not None else native.dispatch(router, protocol, payload),
timeout=timeout,
)
summary: Final = native.extract_summary(protocol, response)
return (
summary
if summary is not None
else _reject(str(payload["model"]), "The provider did not return one complete native compaction block")
)
async def compact_to_fit(
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
) -> Mapping[str, object]:
state: Final = payload.get(_STATE_KEY)
config: Final = state.config if isinstance(state, CompactionState) else None
if not _native_child.get() and (config is None or _client_managed(payload)):
return payload
model: Final = str(payload["model"])
limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9)
budget: Final = limits.available
if budget is None or budget <= 0:
if (
config is not None
and config.model is None
and (limits.window is None or await _count(router, payload) <= limits.window)
):
return payload
_reject(model, "A known input window and a smaller output allowance are required")
if _native_child.get():
if (
not native.compatible_defaults(payload)
or not native.supports_native_compaction(payload)
or await _count(router, payload) > budget
):
_reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget")
return payload
if await _count(router, payload) <= budget:
return payload
if surface is None or config is None or not isinstance(state, CompactionState):
_reject(model, "This request surface cannot be compacted")
key: Final = "input" if surface == "responses" else "messages"
raw_items: Final = payload[key]
if isinstance(raw_items, str):
_reject(model, "A single user input cannot be compacted without changing the latest request")
items: Final = _ITEMS.validate_python(raw_items)
if surface == "responses" and any(
item.get("type", "message") not in ("message", "function_call", "function_call_output") for item in items
):
_reject(model, "Opaque or provider-managed Responses items require client-managed native compaction")
if surface == "responses" and (
any(block.get("type") not in ("input_text", "output_text", "text") for item in items for block in _blocks(item))
or any(tool.get("type") != "function" for tool in _blocks(payload, "tools"))
):
_reject(model, "Only text history and ordinary function tools support portable Responses compaction")
instructions, prefix, tail = _history(items, model)
if any(
item.get("thinking_blocks")
or any(block.get("type") in ("thinking", "redacted_thinking", "compaction") for block in _blocks(item))
for item in items
):
_reject(model, "Native reasoning or compaction blocks require client-managed native compaction")
retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))})
if await _count(router, retained) >= budget:
_reject(model, "Retained instructions, tools and the latest turn leave no room for a summary")
older: Final = _native_prefix(
MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *prefix))}), surface
)
metadata: Final = sanitized_forwardable_call_metadata(
_MAPPING.validate_python(payload.get("litellm_metadata") or payload.get("metadata") or _EMPTY),
"autorouter_compaction",
)
protocol: Final[native.Protocol] = "messages" if surface == "messages" else "chat"
request: Final = MappingProxyType(
{
"model": model,
"messages": older["messages"],
**native.request_kwargs(),
"max_tokens": config.max_tokens,
"stream": False,
"num_retries": 0,
"disable_fallbacks": True,
"timeout": config.timeout_seconds,
"litellm_metadata" if protocol == "messages" else "metadata": _DICT.validate_python(
MappingProxyType(
{
key: value
for key, value in metadata.items()
if key != "user_api_key_auth" or compaction_executor.get() is None
}
)
),
**parent_session_kwargs(payload),
**MappingProxyType({key: older[key] for key in ("system", "tools", "user") if key in older}),
}
)
compactor: Final = await _compactor_model(router, state, request)
child: Final = MappingProxyType({**request, "model": compactor})
identity: Final = hashlib.sha256(
json.dumps(
(protocol, compactor, child["messages"], child.get("system"), child.get("tools")), sort_keys=True
).encode()
).hexdigest()
if state.summary is None:
private: Final = should_redact_message_logging(
_DICT.validate_python(
MappingProxyType(
{
"litellm_params": payload,
"standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(
_DICT.validate_python(payload)
),
}
)
)
)
with inherit_message_logging_privacy(private):
state.summary = (
identity,
asyncio.create_task(_generate_summary(router, protocol, child, config.timeout_seconds)),
)
if state.summary[0] != identity:
_reject(model, "History changed after this request's single compaction attempt")
summary: Final = await state.summary[1]
message: Final = MappingProxyType(
{"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary}
)
compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))})
if await _count(router, compacted) > budget:
_reject(model, "The summary and retained conversation still exceed the selected deployment's budget")
return compacted

View file

@ -24,7 +24,7 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding"]
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "compactor"]
@dataclass(frozen=True, slots=True)
@ -154,6 +154,7 @@ def strategy_router_dependencies(
dict.fromkeys(
tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier"))
+ _named(litellm_params.get("complexity_router_default_model"), "default")
+ _named(_mapping(complexity.get("context_compaction")).get("model"), "compactor")
+ (
_named(classifier.get("model"), "classifier")
if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES

View file

@ -216,11 +216,20 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False):
caller: ToolCaller | None
class CompactionBlock(TypedDict, total=False):
"""Native compaction block, signed for on-demand compaction."""
type: Required[ReadOnly[Literal["compaction"]]]
content: ReadOnly[str | None]
signature: ReadOnly[str]
AnthropicMessagesAssistantMessageValues = (
AnthropicMessagesTextParam
| AnthropicMessagesToolUseParam
| ChatCompletionThinkingBlock
| ChatCompletionRedactedThinkingBlock
| CompactionBlock
)
@ -390,6 +399,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = (
)
class AnthropicCompaction(TypedDict, total=False):
type: Required[ReadOnly[Literal["summarize"]]]
instructions: ReadOnly[str]
class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
max_tokens: int | None
metadata: AnthropicMetadata | dict | None
@ -405,6 +419,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
top_p: float | None
mcp_servers: list[AnthropicMcpServerTool] | None
context_management: dict[str, Any] | None
compaction: ReadOnly[AnthropicCompaction | None]
container: dict[str, Any] | None # Container config with skills for code execution
output_format: AnthropicOutputSchema | None # Structured outputs support
speed: str | None # Fast mode support for Opus models
@ -564,13 +579,6 @@ class ContextManagementResponse(TypedDict, total=False):
applied_edits: list[AppliedEdit]
class CompactionBlock(TypedDict, total=False):
"""Synthesized ``compaction`` content block (compact_20260112)."""
type: Required[Literal["compaction"]]
content: str | None
class UsageIteration(TypedDict, total=False):
"""One sampling iteration's token usage (compact_20260112)."""
@ -743,6 +751,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
WEB_SEARCH_2025_03_05 = "web-search-2025-03-05"
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
COMPACT_2026_01_12 = "compact-2026-01-12"
COMPACT_2026_09_04 = "compact-2026-09-04"
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"

View file

@ -1,3 +1,4 @@
from collections.abc import Sequence
from typing import Any, Literal, TypeAlias
from typing_extensions import NotRequired, ReadOnly, TypedDict
@ -6,8 +7,10 @@ from litellm.types.llms.anthropic import (
AnthropicResponseContentBlockText,
AnthropicResponseContentBlockToolUse,
AnthropicStopDetails,
CompactionBlock,
ContextManagementResponse,
ServerToolUsage,
UsageIteration,
)
@ -56,6 +59,7 @@ AnthropicResponseContentBlock: TypeAlias = (
| AnthropicResponseToolUseBlock
| AnthropicResponseThinkingBlock
| AnthropicResponseRedactedThinkingBlock
| CompactionBlock
)
@ -66,6 +70,7 @@ class AnthropicUsage(TypedDict, total=False):
input_tokens: int
output_tokens: int
iterations: ReadOnly[Sequence[UsageIteration]]
"""
Cache Tokens Used
@ -91,7 +96,9 @@ class AnthropicMessagesResponse(TypedDict, total=False):
id: str
model: str | None # This represents the Model type from Anthropic
role: Literal["assistant"] | None
stop_reason: Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] | None
stop_reason: ReadOnly[
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
]
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
stop_sequence: str | None
type: Literal["message"] | None

View file

@ -173,6 +173,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_output_config: bool | None
supports_image_size: bool | None
supports_anthropic_thinking_payload: ReadOnly[bool | None]
supports_anthropic_compaction: ReadOnly[bool | None]
supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None]
vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None]
bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None
@ -2960,6 +2961,7 @@ RoutingDecisionCause = Literal[
InternalCallOrigin = Literal[
"autorouter_classifier",
"autorouter_compaction",
"shadow_eval_router",
"shadow_eval_judge",
"llm_as_a_judge_guardrail",
@ -3835,6 +3837,7 @@ all_litellm_params = (
agentic_loop_internal_litellm_params
+ [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params]
+ [
"_context_compaction_state",
"metadata",
"litellm_metadata",
"keepalive_seconds",

View file

@ -6120,6 +6120,7 @@ def _get_model_info_helper(
supports_tool_search=_model_info.get("supports_tool_search", None),
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None),
supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None),
supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),
supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None),
supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),

View file

@ -14487,6 +14487,7 @@
"source": "https://docs.anthropic.com/en/docs/about-claude/pricing"
},
"claude-sonnet-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
@ -14526,6 +14527,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-sonnet-4-6": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -14741,6 +14743,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-6": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14779,6 +14782,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-6-20260205": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14816,6 +14820,7 @@
"prompt_cache_min_tokens": 4096
},
"claude-opus-4-7": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14855,6 +14860,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-7-20260416": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -14893,6 +14899,7 @@
"prompt_cache_min_tokens": 2048
},
"claude-fable-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -14932,6 +14939,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-fable-5-1": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
@ -14972,6 +14980,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -15014,6 +15023,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-4-8": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -60334,6 +60344,7 @@
"supports_audio_output": true
},
"claude-mythos-5": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -60373,6 +60384,7 @@
}
},
"claude-mythos-5-1": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
@ -60413,6 +60425,7 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-mythos-preview": {
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,

View file

@ -815,6 +815,9 @@
"supports_adaptive_thinking": {
"type": "boolean"
},
"supports_anthropic_compaction": {
"type": "boolean"
},
"supports_anthropic_thinking_payload": {
"type": "boolean"
},

View file

@ -12,6 +12,7 @@ ALLOWED_FILES = [
"../../litellm/llms/custom_httpx/httpx_handler.py",
"../../litellm/llms/openai/common_utils.py",
"../../litellm/experimental_mcp_client/client.py",
"../../litellm/proxy/native_compaction.py",
# when running on ci/cd
"./litellm/__init__.py",
"./litellm/llms/custom_httpx/http_handler.py",
@ -22,6 +23,7 @@ ALLOWED_FILES = [
"./litellm/llms/custom_httpx/httpx_handler.py",
"./litellm/llms/openai/common_utils.py",
"./litellm/experimental_mcp_client/client.py",
"./litellm/proxy/native_compaction.py",
]
warning_msg = "this is a serious violation that can impact latency. Creating Async clients per request can add +500ms per request"

View file

@ -1,8 +1,12 @@
from types import MappingProxyType
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
inherit_message_logging_privacy,
initialize_standard_callback_dynamic_params,
iter_client_callback_metadata_dicts,
)
@ -189,6 +193,20 @@ def test_empty_kwargs_returns_empty_params():
assert dict(params) == {}
@pytest.mark.parametrize("child_privacy", (False, True))
def test_inherited_privacy_only_strengthens_child_and_resets(child_privacy: bool) -> None:
kwargs: Final = TypeAdapter(dict[str, object]).validate_python(
MappingProxyType({"turn_off_message_logging": child_privacy})
)
with inherit_message_logging_privacy(False):
assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy
with inherit_message_logging_privacy(True), inherit_message_logging_privacy(False):
params: Final = initialize_standard_callback_dynamic_params(kwargs)
assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy
assert params["turn_off_message_logging"] is True
assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None
def test_newrelic_callback_params_are_not_extracted_from_request_kwargs():
kwargs = {
"newrelic_api_key": "caller-key",

View file

@ -1,8 +1,12 @@
import pytest
import json
from typing import Final
from unittest.mock import MagicMock, patch
import httpx
import pytest
import respx
import litellm
from litellm.constants import (
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
@ -3771,6 +3775,69 @@ def test_multiple_compaction_blocks():
assert compaction_blocks[1]["content"] == "Second summary..."
@pytest.mark.parametrize("messages_api,gateway,native_endpoint", [
(False, False, False), (True, False, False), (False, True, False), (True, True, False), (True, True, True),
])
async def test_native_compaction_wire_roundtrip(
messages_api: bool, gateway: bool, native_endpoint: bool,
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False)
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
operation: Final = {"type": "summarize", "instructions": "Keep identifiers"}
usage: Final = {"input_tokens": 0, "output_tokens": 0,
"iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}]}
chat_wire: Final = gateway and not native_endpoint
base: Final = "https://gateway.test/v1" if gateway else "https://api.anthropic.com/v1"
route: Final = respx_mock.post(f"{base}/{'chat/completions' if chat_wire else 'messages'}")
def respond(request: httpx.Request) -> httpx.Response:
payload: Final = json.loads(request.content)
assert len(request.headers.get_list("anthropic-beta")) == 1
assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == {
"compact-2026-09-04", "interleaved-thinking-2025-05-14",
}
if "compaction" in payload:
assert payload["compaction"] == operation
else:
assert payload["messages"][0] == {"role": "assistant", "content": [block]}
body: Final = (
{"id": "chatcmpl_compact", "object": "chat.completion", "created": 1, "model": "claude-sonnet-5",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "",
"provider_specific_fields": {"compaction_blocks": [block]}}}],
"usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268}}
if chat_wire else
{"id": "msg_compact", "type": "message", "role": "assistant", "model": "claude-sonnet-5",
"content": [block], "stop_reason": "compaction", "usage": usage}
)
return httpx.Response(200, json=body)
route.mock(side_effect=respond)
call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion
params: Final = dict(
model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", max_tokens=512,
api_base=base if gateway else "https://api.anthropic.com",
extra_headers={"Anthropic-Beta": f"interleaved-thinking-2025-05-14{',compact-2026-09-04' if gateway else ''}"},
model_info={"supported_endpoints": ["/v1/messages"]} if native_endpoint else {},
)
response: Final = await call(
messages=[{"role": "user", "content": "Remember identifiers"}], compaction=operation, **params
)
message: Final = response if messages_api else response.choices[0].message.model_dump()
blocks: Final = message["content"] if messages_api else message["provider_specific_fields"]["compaction_blocks"]
assert blocks == [block]
if messages_api:
assert response["stop_reason"] == "compaction"
if not chat_wire:
assert response["usage"] == usage
if not gateway:
replay: Final = {"role": "assistant", "content": blocks} if messages_api else message
await call(messages=[replay, {"role": "user", "content": "Continue"}], **params)
assert route.call_count == (1 if gateway else 2)
def test_compaction_block_request_transformation():
"""
Test that compaction blocks from provider_specific_fields are correctly

View file

@ -60,6 +60,20 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None:
assert result["usage"]["input_tokens"] == 10
@pytest.mark.parametrize("text,count,expected_stop", [
("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"),
(" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"),
])
def test_native_compaction_response_roundtrip(text: str | None, count: int, expected_stop: str) -> None:
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
message: Final = Message(content=text, provider_specific_fields={"compaction_blocks": [block] * count})
response: Final = ModelResponse(choices=[Choices(message=message, finish_reason="length")], usage=Usage())
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
expected_text: Final = [{"type": "text", "text": text}] if text is not None and (text != "" or not count) else []
assert result["content"] == [*([block] * count), *expected_text]
assert result["stop_reason"] == expected_stop
def test_translate_chat_refusal_to_anthropic_response():
response = ModelResponse(
id="chatcmpl-refusal",

View file

@ -0,0 +1,145 @@
import asyncio
from collections.abc import Awaitable, Mapping
from types import MappingProxyType
from typing import Final, Literal
import pytest
from fastapi import FastAPI, Request
from pydantic import TypeAdapter
from litellm.caching.caching import DualCache
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.proxy import common_request_processing, proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash, get_request_stash
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.router import Router
from litellm.router_strategy.complexity_router.context_compaction import compaction_executor, reject_recursive_compactor
from litellm.types.utils import ModelResponse
_HEADERS: Final = (
(b"authorization", b"Bearer sk-compaction-fixture"), (b"cookie", b"session=fixture"),
(b"content-length", b"99999"), (b"x-litellm-call-id", b"parent"),
(b"litellm-disable-message-redaction", b"true"), (b"x-litellm-num-retries", b"8"),
(b"X-LiteLLM-Timeout", b"600"), (b"x-litellm-stream-timeout", b"500"),
)
async def _child(protocol: Literal["chat", "messages"] = "chat", forged: bool = False) -> Mapping[str, object]:
executor: Final = compaction_executor.get()
assert executor is not None
payload: Final = TypeAdapter(Mapping[str, object]).validate_json(
b'{"model":"compactor","messages":[{"role":"user","content":"history"}],'
b'"num_retries":0,"timeout":7,"stream_timeout":7,"disable_fallbacks":true,"stream":false,'
b'"metadata":{"turn_off_message_logging":true}}'
)
return await executor(protocol, MappingProxyType({
"litellm_metadata" if protocol == "messages" and key == "metadata" else key: value
for key, value in payload.items() if forged or key != "metadata"
}))
def _request(app: FastAPI) -> Request:
return Request(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({
"type": "http", "app": app, "scheme": "https", "server": ("proxy.test", 443),
"path": "/gateway/parent", "root_path": "/gateway", "query_string": b"parent=1",
"client": ("192.0.2.1", 4321), "headers": _HEADERS,
})))
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ("chat", "messages"))
async def test_child_preserves_credentials_and_isolates_context(protocol: Literal["chat", "messages"]) -> None:
app: Final = FastAPI()
stash: Final = get_or_create_request_stash()
@app.post("/v1/chat/completions" if protocol == "chat" else "/v1/messages")
async def endpoint(request: Request) -> Mapping[str, object]:
assert get_request_stash() is None and compaction_executor.get() is None
assert request.client == ("192.0.2.1", 4321) and request.url.scheme == "https"
assert request.scope["root_path"] == "/gateway" and request.cookies["session"] == "fixture"
assert request.headers["authorization"] == "Bearer sk-compaction-fixture" and not request.query_params
assert "x-litellm-call-id" not in request.headers
assert "litellm-disable-message-redaction" not in request.headers
assert int(request.headers["content-length"]) == len(await request.body())
with pytest.raises(BadRequestError, match="regular model group"):
reject_recursive_compactor("auto-router")
return MappingProxyType({"summary": "compacted"})
with inherit_message_logging_privacy(True):
assert (await with_proxy_compaction_executor(_child(protocol), _request(app)))["summary"] == "compacted"
assert get_request_stash() is stash and compaction_executor.get() is None
reject_recursive_compactor("auto-router")
@pytest.mark.asyncio
@pytest.mark.parametrize("protocol", ("chat", "messages"))
@pytest.mark.parametrize("policy", ("allowed", "denied", "forged"))
async def test_real_proxy_child_auth_privacy_and_body_policy(
monkeypatch: pytest.MonkeyPatch, protocol: Literal["chat", "messages"], policy: str,
) -> None:
cache: Final = DualCache()
token: Final = proxy_server.hash_token("sk-compaction-fixture")
await cache.async_set_cache(key=token, value=UserAPIKeyAuth.model_validate(
MappingProxyType({"token": token, "models": ("answer",) if policy == "denied" else ("compactor",)})
))
dispatched: Final = asyncio.Event()
async def route(
data: Mapping[str, object], llm_router: Router | None, user_model: str | None,
route_type: str, user_api_key_dict: UserAPIKeyAuth | None,
) -> Awaitable[ModelResponse]:
dispatched.set()
assert policy == "allowed"
assert (data["num_retries"], data["timeout"], data["stream_timeout"]) == (0, 7, 7)
assert data["disable_fallbacks"] is True and data["stream"] is False
logging: Final = data["litellm_logging_obj"]
assert isinstance(logging, Logging)
assert logging.standard_callback_dynamic_params.get("turn_off_message_logging") is True
assert should_redact_message_logging(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({
"litellm_params": data, "standard_callback_dynamic_params": logging.standard_callback_dynamic_params,
})))
return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor"))
monkeypatch.setattr(proxy_server.app, "dependency_overrides", {})
monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture")
monkeypatch.setattr(proxy_server, "prisma_client", object())
monkeypatch.setattr(proxy_server, "user_api_key_cache", cache)
monkeypatch.setattr(proxy_server, "llm_router", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(common_request_processing, "route_request", route)
with inherit_message_logging_privacy(True):
call: Final = with_proxy_compaction_executor(_child(protocol, policy == "forged"), _request(proxy_server.app))
if policy == "allowed":
assert (await call)["id"] == "private-summary"
else:
status: Final = 403 if policy == "denied" else 401
with pytest.raises(BadRequestError, match=rf"child request failed \(HTTP {status}\)"):
await call
assert dispatched.is_set() is (policy == "allowed")
assert compaction_executor.get() is None
@pytest.mark.asyncio
async def test_cancelling_parent_cancels_and_drains_child() -> None:
app: Final = FastAPI()
started: Final = asyncio.Event()
stopped: Final = asyncio.Event()
@app.post("/v1/chat/completions")
async def endpoint() -> None:
started.set()
try:
await asyncio.Event().wait()
finally:
stopped.set()
parent: Final = asyncio.create_task(with_proxy_compaction_executor(_child(), _request(app)))
await asyncio.wait_for(started.wait(), timeout=5)
parent.cancel()
with pytest.raises(asyncio.CancelledError):
await parent
assert stopped.is_set() and compaction_executor.get() is None

View file

@ -0,0 +1,393 @@
import asyncio
import json
from collections.abc import Iterator, Mapping
from copy import deepcopy
from typing import Final, Literal
import httpx
import pytest
import respx
import litellm
from litellm.llms.anthropic import compaction as native
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
from litellm.router_strategy.complexity_router.context_compaction import (
CompactionState,
Surface,
arm_compaction,
compact_to_fit,
compaction_executor,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
pytestmark: Final = [pytest.mark.asyncio, pytest.mark.usefixtures("local_model_cost_map")]
SCHEMA: Final = {"type": "object", "properties": {"code": {"type": "string"}}}
@pytest.fixture(autouse=True)
def native_catalog(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setenv("LITELLM_LICENSE", "")
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False)
monkeypatch.setitem(litellm.model_cost, "summary-fixture", {
"litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 32000,
"max_output_tokens": 4096, "supports_anthropic_compaction": True,
})
def make_router(
window: int | None = 512, settings: Mapping[str, object] | None = None, *, compactor_window: int = 32000,
conflict: bool = False, output: int | None = 64,
) -> litellm.Router:
config: Final = {
"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"},
"keyword_tier_rules": [{"keywords": ["answer", "tail result"], "tier": "SIMPLE"}],
"enable_context_window_escalation": False, "max_tokens_from_tier_model": False,
**(settings or {}),
}
return litellm.Router(model_list=[
{"model_name": "auto", "litellm_params": {
"model": "auto_router/complexity_router", "complexity_router_config": config,
}},
{"model_name": "small", "litellm_params": {
"model": "openai/arbitrary-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", "max_retries": 0,
}, "model_info": {"id": "pinned-answer", "max_input_tokens": window, "max_output_tokens": output}},
{"model_name": "large", "litellm_params": {
"model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "compact-test",
**({"stop": ["deployment policy"]} if conflict else {}),
}, "model_info": {"id": "native-compactor", "max_input_tokens": compactor_window, "max_output_tokens": 4096}},
{"model_name": "backup", "litellm_params": {
"model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "backup-test",
}, "model_info": {"id": "backup-compactor", "max_input_tokens": 32000, "max_output_tokens": 4096}},
], enable_pre_call_checks=True, num_retries=0, disable_cooldowns=True,
retry_policy={"InternalServerErrorRetries": 1})
def exchange(surface: Surface, phase: str) -> list[dict[str, object]]:
identifier: Final = f"{phase}-call"
result: Final = f"{phase} result"
if surface == "responses":
return [
{"type": "function_call", "call_id": identifier, "name": "lookup", "arguments": "{}"},
{"type": "function_call_output", "call_id": identifier, "output": result},
]
if surface == "messages":
return [
{"role": "assistant", "content": [{"type": "tool_use", "id": identifier, "name": "lookup", "input": {}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": identifier, "content": result}]},
]
return [
{"role": "assistant", "tool_calls": [
{"id": identifier, "type": "function", "function": {"name": "lookup", "arguments": "{}"}},
]},
{"role": "tool", "tool_call_id": identifier, "content": result},
]
def history(surface: Surface) -> dict[str, object]:
conversation: Final = [
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * 150},
{"role": "assistant", "content": "Recorded"},
*exchange(surface, "prefix"),
{"role": "user", "content": "Answer with the project code"},
*exchange(surface, "tail"),
]
function: Final = {"name": "lookup", "parameters": SCHEMA}
if surface == "responses":
return {"instructions": "Keep the code exact", "tools": [{"type": "function", **function}], "input": [
{"role": "developer", "content": "Retain the original spelling"}, *conversation,
]}
if surface == "messages":
return {"system": "Keep the code exact", "tools": [{"name": "lookup", "input_schema": SCHEMA}], "messages": [
*conversation,
]}
return {"tools": [{"type": "function", "function": function}], "messages": [
{"role": "system", "content": "Keep the code exact"}, *conversation,
]}
def native_reply(summary: str = "Project code MAPLE-47", signed: bool = True, truncated: bool = False) -> httpx.Response:
return httpx.Response(200, json={
"id": "msg_compact", "type": "message", "role": "assistant", "model": "summary-fixture",
"content": [{"type": "compaction", "content": summary, **({"signature": "native-signature"} if signed else {})}],
"stop_reason": "max_tokens" if truncated else "compaction", "usage": {"input_tokens": 0, "output_tokens": 0, "iterations": [
{"type": "compaction", "input_tokens": 1200, "output_tokens": 20},
]},
})
def answer_reply(request: httpx.Request) -> httpx.Response:
payload: Final = json.loads(request.content)
assert payload["model"] == "arbitrary-answer"
assert request.headers["authorization"] == "Bearer answer-test"
if request.url.path.endswith("responses"):
return httpx.Response(200, json={
"id": "resp_answer", "object": "response", "created_at": 0, "status": "completed",
"model": payload["model"], "output": [{"id": "msg_answer", "type": "message", "role": "assistant",
"status": "completed", "content": [{"type": "output_text", "text": "MAPLE-47", "annotations": []}]}],
"usage": {"input_tokens": 60, "output_tokens": 8, "total_tokens": 68},
})
return httpx.Response(200, json={
"id": "answer", "object": "chat.completion", "created": 0, "model": payload["model"],
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "MAPLE-47"}}],
"usage": {"prompt_tokens": 60, "completion_tokens": 8, "total_tokens": 68},
})
@pytest.fixture
def wire() -> Iterator[tuple[respx.Route, respx.Route]]:
with respx.mock(assert_all_called=False) as transport:
compactor: Final = transport.post("https://compact.test/v1/messages").mock(return_value=native_reply())
answer: Final = transport.route(method="POST", host="answer.test").mock(side_effect=answer_reply)
yield compactor, answer
async def invoke(router: litellm.Router, surface: Surface, payload: Mapping[str, object], retries: int = 0) -> object:
if surface == "responses":
return await router.aresponses(model="auto", max_output_tokens=64, num_retries=retries, **payload)
if surface == "messages":
return await router.aanthropic_messages(model="auto", max_tokens=64, num_retries=retries, **payload)
return await router.acompletion(model="auto", max_tokens=64, num_retries=retries, **payload)
@pytest.mark.parametrize("surface", ["chat", "messages", "responses"])
@pytest.mark.parametrize("near", [False, True])
@pytest.mark.parametrize("configured", [False, True])
async def test_all_surfaces_compact_and_keep_selected_answerer(
wire: tuple[respx.Route, respx.Route], surface: Surface, near: bool, configured: bool,
) -> None:
payload: Final = history(surface)
original: Final = deepcopy(payload)
counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload)
window: Final = int((counted + 32) / ContextCompactionConfig().trigger_ratio) + 1 if near else 512
assert (counted < window) is near
settings: Final = {"context_compaction": {"model": "large", "max_tokens": 512}} if configured else {}
router: Final = make_router(window, settings)
captured: Final = asyncio.Queue[Mapping[str, object]]()
compactor, answer = wire
retry: Final = near and configured
def answer_after_retry(request: httpx.Request) -> httpx.Response:
return httpx.Response(500, json={"error": {"message": "retry answer"}}) if answer.call_count == 0 else answer_reply(request)
if retry:
answer.mock(side_effect=answer_after_retry)
async def execute(protocol: native.Protocol, request: Mapping[str, object]) -> Mapping[str, object]:
result: Final = await native.dispatch(router, protocol, request)
captured.put_nowait(result)
return result
token: Final = compaction_executor.set(execute)
try:
response: Final = await invoke(router, surface, payload, retries=int(retry))
finally:
compaction_executor.reset(token)
assert compactor.call_count == captured.qsize() == 1
assert answer.call_count == router.total_calls["openai/arbitrary-answer"] == 1 + int(retry)
compact_request: Final = compactor.calls[0].request
compact_body: Final = json.loads(compact_request.content)
answer_body: Final = json.loads(answer.calls[0].request.content)
assert answer_body == json.loads(answer.calls[-1].request.content)
assert compact_body["model"] == "summary-fixture" and compact_body["compaction"] == {"type": "summarize"}
assert ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value in compact_request.headers["anthropic-beta"].split(",")
assert compact_body["max_tokens"] == (512 if configured else ContextCompactionConfig().max_tokens)
assert "Background detail" in str(compact_body) and "tail-call" not in str(compact_body)
assert "prefix-call" in str(compact_body) and "prefix result" in str(compact_body)
assert "Keep the code exact" in str(compact_body["system"])
assert compact_body["tools"][0]["input_schema"] == SCHEMA
assert "MAPLE-47" in str(answer_body) and "Background detail" not in str(answer_body)
assert "prefix-call" not in str(answer_body)
assert "native-signature" not in str(answer_body) and "compaction" not in answer_body
assert "tail-call" in str(answer_body) and "tail result" in str(answer_body)
assert "MAPLE-47" in str(response)
usage: Final = captured.get_nowait()["usage"]
if surface == "messages":
assert usage["iterations"][0]["input_tokens"] == 1200 and usage["iterations"][0]["output_tokens"] == 20
else:
assert usage["prompt_tokens"] == 1200 and usage["completion_tokens"] == 20
if surface == "responses":
assert answer_body["input"][-3:] == original["input"][-3:]
assert answer_body["input"][0] == original["input"][0]
assert answer_body["instructions"] == original["instructions"] and answer_body["tools"] == original["tools"]
assert payload == original
@pytest.mark.parametrize("surface", ["chat", "messages", "responses"])
@pytest.mark.parametrize("mode", ["fitting", "false", "null"])
async def test_fitting_and_disabled_requests_do_not_compact(
wire: tuple[respx.Route, respx.Route], surface: Surface, mode: str,
) -> None:
settings: Final = {} if mode == "fitting" else {"context_compaction": False if mode == "false" else None}
payload: Final = history(surface)
original: Final = deepcopy(payload)
counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload)
await invoke(make_router(20000 if mode == "fitting" else counted + 64, settings), surface, payload)
compactor, answer = wire
assert compactor.call_count == 0 and answer.call_count == 1
assert "Background detail" in answer.calls[0].request.content.decode()
assert payload == original
@pytest.mark.parametrize(
("surface", "failure"),
[(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")]
+ [("messages", "truncated")],
)
async def test_bad_native_result_never_reaches_answerer(
wire: tuple[respx.Route, respx.Route], surface: Surface, failure: str,
) -> None:
compactor, answer = wire
reply: Final = httpx.Response(500, json={"error": {"type": "api_error", "message": "failed"}}) if failure == "provider" else native_reply(
"too large " * 2000 if failure == "oversized" else "MAPLE-47", signed=failure != "unsigned",
truncated=failure == "truncated",
)
compactor.mock(return_value=reply)
with pytest.raises((litellm.BadRequestError, litellm.InternalServerError)):
await invoke(make_router(), surface, history(surface))
assert compactor.call_count == 1 and answer.call_count == 0
@pytest.mark.parametrize("failure", ["item", "content", "tools", "unclosed", "missing", "duplicate", "instructions", "retained"])
@pytest.mark.parametrize("needed", [False, True])
async def test_unsafe_responses_reject_only_when_compaction_needed(
wire: tuple[respx.Route, respx.Route], failure: str, needed: bool,
) -> None:
payload: Final = history("responses")
extra: Final = {
"item": [{"type": "computer_call", "call_id": "opaque-tool"}],
"content": [{"role": "assistant", "content": [{"type": "refusal", "refusal": "cannot"}]}],
"unclosed": [{"type": "function_call", "call_id": "unclosed", "name": "lookup", "arguments": "{}"}],
"missing": [{"type": "function_call", "name": "lookup", "arguments": "{}"}],
"duplicate": exchange("responses", "prefix"),
"instructions": [{"role": "developer", "content": "Changed instructions"}],
}
request: Final = {
**payload, "input": [*payload["input"][:3], *extra.get(failure, []), *payload["input"][3:]],
**({"tools": [{"type": "computer_use_preview", "display_width": 800, "display_height": 600}]} if failure == "tools" else {}),
**({"instructions": "Keep every instruction " * 600} if failure == "retained" else {}),
}
if needed:
with pytest.raises(litellm.BadRequestError, match="Context compaction"):
await invoke(make_router(), "responses", request)
assert all(route.call_count == 0 for route in wire)
else:
await invoke(make_router(20000), "responses", request)
compactor, answer = wire
assert compactor.call_count == 0 and answer.call_count == 1
@pytest.mark.parametrize("owned", [
{"previous_response_id": "resp_parent"}, {"conversation": "conv_parent"},
{"context_management": [{"type": "compaction", "compact_threshold": 1000}]}, {"compaction": {"type": "summarize"}},
{"input": [{"type": "reasoning", "encrypted_content": "opaque"}]},
{"input": [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "prior reasoning"}]}]},
{"input": [{"type": "compaction", "encrypted_content": "opaque"}]},
{"input": [{"type": "item_reference", "id": "item_parent"}]},
{"input": [{"role": "assistant", "content": "visible", "encrypted_content": "opaque"}]},
{"input": [{"role": "user", "content": [{"type": "encrypted_content", "encrypted_content": "opaque"}]}]},
])
@pytest.mark.parametrize("arm_first", [False, True])
async def test_client_owned_history_bypasses_compaction(
wire: tuple[respx.Route, respx.Route], owned: Mapping[str, object], arm_first: bool,
) -> None:
router: Final = make_router()
deployment: Final = router.get_deployment(model_id="pinned-answer")
assert deployment is not None
state: Final = CompactionState()
request: Final = {**history("responses"), **owned, "model": "small", "max_tokens": 64, "_context_compaction_state": state}
original: Final = deepcopy({key: value for key, value in request.items() if key != "_context_compaction_state"})
before_defaults: Final = {"_context_compaction_state": state} if arm_first else request
arm_compaction(before_defaults, ContextCompactionConfig(), ("large",))
result: Final = await compact_to_fit(router, deployment.model_dump(), request, "responses")
assert result is request
assert {key: value for key, value in result.items() if key != "_context_compaction_state"} == original
assert all(route.call_count == 0 for route in wire)
@pytest.mark.parametrize("case", ["conflicting_defaults", "small_window", "capability_false", "capability_missing"])
async def test_automatic_compactor_skips_conflicts_and_requires_capacity_and_capability(
wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, case: str,
) -> None:
if case.startswith("capability"):
metadata: Final = {key: value for key, value in litellm.model_cost["summary-fixture"].items()
if key != "supports_anthropic_compaction"}
monkeypatch.setitem(litellm.model_cost, "summary-fixture", {
**metadata, **({"supports_anthropic_compaction": False} if case == "capability_false" else {}),
})
conflict: Final = case == "conflicting_defaults"
settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "backup", "REASONING": "backup"}} if conflict else {}
router: Final = make_router(settings=settings, conflict=conflict, compactor_window=512 if case == "small_window" else 32000)
if conflict:
await invoke(router, "chat", history("chat"))
compactor, answer = wire
assert compactor.call_count == answer.call_count == 1
assert compactor.calls[0].request.headers["x-api-key"] == "backup-test"
assert "stop_sequences" not in json.loads(compactor.calls[0].request.content)
else:
with pytest.raises(litellm.BadRequestError, match="No configured compactor"):
await invoke(router, "chat", history("chat"))
assert all(route.call_count == 0 for route in wire)
@pytest.mark.parametrize("unknown_output", [False, True])
@pytest.mark.parametrize("overflow", [False, True])
async def test_unusable_output_budget_still_enforces_known_input_window(
wire: tuple[respx.Route, respx.Route], unknown_output: bool, overflow: bool,
) -> None:
payload: Final = history("chat")
counted: Final = make_router(output=None)._count_pre_call_check_tokens(payload["messages"], None, payload)
window: Final = 512 if overflow else counted + 64
output: Final = None if unknown_output else window
router: Final = make_router(window, output=output)
deployment: Final = router.get_deployment(model_id="pinned-answer")
assert deployment is not None
state: Final = CompactionState(config=ContextCompactionConfig(), candidates=("large",))
request: Final = {**payload, "model": "small", "max_tokens": output, "_context_compaction_state": state}
if overflow:
with pytest.raises(litellm.BadRequestError, match="known input window"):
await compact_to_fit(router, deployment.model_dump(), request, "chat")
else:
assert await compact_to_fit(router, deployment.model_dump(), request, "chat") is request
assert all(route.call_count == 0 for route in wire)
@pytest.mark.parametrize("outcome", ["success", "timeout", "cancel"])
async def test_retry_reuses_summary_or_terminal_cancellation(outcome: Literal["success", "timeout", "cancel"]) -> None:
router: Final = make_router()
deployment: Final = router.get_deployment(model_id="pinned-answer")
assert deployment is not None
state: Final = CompactionState(config=ContextCompactionConfig(model="large", max_tokens=512, timeout_seconds=0.02))
request: Final = {**history("messages"), "model": "small", "max_tokens": 64, "_context_compaction_state": state}
calls: Final = asyncio.Queue[None]()
started: Final = asyncio.Event()
stopped: Final = asyncio.Event()
async def execute(protocol: native.Protocol, payload: Mapping[str, object]) -> Mapping[str, object]:
calls.put_nowait(None)
started.set()
try:
return native_reply().json() if outcome == "success" else await asyncio.Future[Mapping[str, object]]()
finally:
stopped.set()
token: Final = compaction_executor.set(execute)
try:
first: Final = asyncio.create_task(compact_to_fit(router, deployment.model_dump(), request, "messages"))
await asyncio.wait_for(started.wait(), timeout=2)
if outcome == "cancel":
first.cancel()
if outcome == "success":
assert await first == await compact_to_fit(router, deployment.model_dump(), request, "messages")
changed: Final = {**request, "messages": [{"role": "user", "content": "new history"}, *request["messages"]]}
with pytest.raises(litellm.BadRequestError, match="History changed"):
await compact_to_fit(router, deployment.model_dump(), changed, "messages")
else:
error: Final = asyncio.CancelledError if outcome == "cancel" else asyncio.TimeoutError
with pytest.raises(error):
await first
with pytest.raises(error):
await compact_to_fit(router, deployment.model_dump(), request, "messages")
assert calls.qsize() == 1 and stopped.is_set()
finally:
compaction_executor.reset(token)

View file

@ -15035,7 +15035,9 @@ class TestHealthFallbackDispatch:
) -> None:
from litellm.types.router import RouterRateLimitError
router: Final = self._router(config={"tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}})
router: Final = self._router(
config={"context_compaction": False, "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}}
)
router.add_deployment(
Deployment(
model_name="large",
@ -15112,7 +15114,9 @@ class TestHealthFallbackDispatch:
@pytest.mark.asyncio
@pytest.mark.parametrize("default_fits", [True, False])
async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None:
router: Final = self._router(config={"modality_routing": True, "tiers": {"SIMPLE": "primary"}})
router: Final = self._router(
config={"context_compaction": False, "modality_routing": True, "tiers": {"SIMPLE": "primary"}}
)
for deployment in router.model_list:
deployment["model_info"]["supports_vision"] = deployment["model_name"] == "fallback"
deployment["model_info"]["max_input_tokens"] = 10000 if default_fits else 10

View file

@ -82,6 +82,7 @@ class TestAnthropicBetaHeadersFiltering:
filtered = filter_and_transform_beta_headers(
beta_headers=all_headers, provider=provider
)
assert ("compact-2026-09-04" in filtered) is (provider == "anthropic")
for header in unsupported_headers:
assert (

View file

@ -178,6 +178,14 @@ def test_potential_model_names_keeps_provider_prefixed_candidate():
assert bare["provider_prefixed_model_name"] == bare["combined_model_name"] == "perplexity/glm-5.2"
@pytest.mark.parametrize("capability", [True, False, None])
def test_get_model_info_anthropic_compaction(
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, capability: bool | None
) -> None:
monkeypatch.setitem(litellm.model_cost["claude-sonnet-5"], "supports_anthropic_compaction", capability)
assert litellm.get_model_info("claude-sonnet-5")["supports_anthropic_compaction"] is capability
def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local_model_cost_map):
info = litellm.get_model_info(model="ft:gpt-4o-2024-08-06:my-org::abc123", custom_llm_provider="openai")
assert info["key"] == "ft:gpt-4o-2024-08-06"
@ -855,6 +863,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"source": {"type": "string"},
"comment": {"type": "string"},
"supports_assistant_prefill": {"type": "boolean"},
"supports_anthropic_compaction": {"type": "boolean"},
"supports_audio_input": {"type": "boolean"},
"supports_audio_output": {"type": "boolean"},
"gemini_native_audio": {"type": "boolean"},

View file

@ -27419,6 +27419,26 @@ export interface components {
*/
pattern_type: "prebuilt" | "regex";
};
/** ContextCompactionConfig */
ContextCompactionConfig: {
/**
* Max Tokens
* @default 4096
*/
max_tokens: number;
/** Model */
model?: string | null;
/**
* Timeout Seconds
* @default 120
*/
timeout_seconds: number;
/**
* Trigger Ratio
* @default 0.9
*/
trigger_ratio: number;
};
/**
* CoordinationRedisNode
* @description A single startup node of a cluster-mode Redis used for proxy coordination.
@ -36604,6 +36624,11 @@ export interface components {
* @description Keywords indicating code-related content
*/
code_keywords?: string[] | null;
/**
* Context Compaction
* @description Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.
*/
context_compaction?: components["schemas"]["ContextCompactionConfig"] | false;
/**
* Context Window Escalation Buffer
* @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens.