mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(router): compact long histories for the selected complexity tier
This commit is contained in:
parent
b946d12ffd
commit
7ae9ebc6c6
33 changed files with 1525 additions and 45 deletions
|
|
@ -7,6 +7,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,24 @@
|
|||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
|
||||
|
||||
_CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
|
||||
_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
|
||||
"inherited_message_logging_disabled", default=False
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
|
||||
token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_inherited_message_logging_disabled.reset(token)
|
||||
|
||||
|
||||
def iter_client_callback_metadata_dicts(
|
||||
|
|
@ -138,7 +152,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple
|
|||
|
||||
|
||||
def initialize_standard_callback_dynamic_params(
|
||||
kwargs: dict | None = None,
|
||||
kwargs: dict[str, object] | None = None,
|
||||
) -> StandardCallbackDynamicParams:
|
||||
"""
|
||||
Initialize the standard callback dynamic params from the kwargs
|
||||
|
|
@ -174,4 +188,10 @@ def initialize_standard_callback_dynamic_params(
|
|||
if param in _trusted_overlay_callback_params:
|
||||
standard_callback_dynamic_params[param] = trusted_value
|
||||
|
||||
if _inherited_message_logging_disabled.get():
|
||||
private_params: Final[StandardCallbackDynamicParams] = {
|
||||
**standard_callback_dynamic_params,
|
||||
"turn_off_message_logging": True,
|
||||
}
|
||||
return private_params
|
||||
return standard_callback_dynamic_params
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ from ..common_utils import (
|
|||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
)
|
||||
|
||||
|
|
@ -1780,13 +1781,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers: Dictionary of headers to update
|
||||
beta_value: The beta header value to add
|
||||
"""
|
||||
existing_beta: Final = headers.get("anthropic-beta")
|
||||
if existing_beta is None:
|
||||
headers["anthropic-beta"] = beta_value
|
||||
return
|
||||
existing_values: Final = [beta.strip() for beta in existing_beta.split(",")]
|
||||
if beta_value not in existing_values:
|
||||
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
|
||||
existing_values: Final = tuple(
|
||||
beta.strip()
|
||||
for key, value in headers.items()
|
||||
if key.lower() == "anthropic-beta"
|
||||
for beta in value.split(",")
|
||||
if beta.strip()
|
||||
)
|
||||
for key in tuple(headers):
|
||||
if key.lower() == "anthropic-beta":
|
||||
headers.pop(key)
|
||||
headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value)))
|
||||
|
||||
def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None:
|
||||
"""
|
||||
|
|
@ -1824,7 +1829,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
)
|
||||
|
||||
def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict:
|
||||
def update_headers_with_optional_anthropic_beta(
|
||||
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
|
||||
) -> dict:
|
||||
"""Update headers with optional anthropic beta."""
|
||||
|
||||
# Skip adding beta headers for Vertex requests
|
||||
|
|
@ -1833,6 +1840,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if is_vertex_request:
|
||||
return headers
|
||||
|
||||
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
|
||||
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
_tools: Final = optional_params.get("tools", [])
|
||||
for tool in _tools:
|
||||
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
|
||||
|
|
@ -1929,8 +1939,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
custom_llm_provider=self._resolved_provider,
|
||||
)
|
||||
|
||||
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
|
||||
|
||||
# === Tool-name sanitization (single chokepoint) ===
|
||||
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
|
||||
# sanitize *here* -- not in map_openai_params -- because:
|
||||
|
|
@ -1977,6 +1985,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
message=f"{e}\nReceived Messages={messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
headers=headers, optional_params=optional_params, messages=anthropic_messages
|
||||
)
|
||||
|
||||
## Auto-strip advisor blocks from history if advisor tool is absent.
|
||||
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
|
||||
_all_tools: Final = optional_params.get("tools") or []
|
||||
|
|
|
|||
|
|
@ -77,6 +77,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
|||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
||||
|
||||
def requires_native_compaction_beta(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and bool(block.get("signature"))
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, (list, tuple))
|
||||
for block in content
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
|
|||
111
litellm/llms/anthropic/compaction.py
Normal file
111
litellm/llms/anthropic/compaction.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, NoReturn
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_CHAT_MESSAGES: Final = TypeAdapter(list[AllMessageValues])
|
||||
_SEQUENCE: Final = TypeAdapter(tuple[object, ...])
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
CLIENT_COMPACTION_PARAMS: Final = ("context_management", "compaction")
|
||||
|
||||
|
||||
def request_kwargs() -> Mapping[str, object]:
|
||||
operation: Final[AnthropicCompaction] = {"type": "summarize"}
|
||||
return MappingProxyType(
|
||||
{
|
||||
"compaction": operation,
|
||||
"extra_headers": {"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def supports_native_compaction(params: Mapping[str, object]) -> bool:
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
if params.get("custom_llm_provider") not in (None, "anthropic", "openai"):
|
||||
return False
|
||||
model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/")
|
||||
try:
|
||||
return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _reject(model: str, reason: str) -> NoReturn:
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="")
|
||||
|
||||
|
||||
def _objects(value: object) -> tuple[Mapping[str, object], ...]:
|
||||
return (
|
||||
tuple(
|
||||
_MAPPING.validate_python(block) for block in _SEQUENCE.validate_python(value) if isinstance(block, Mapping)
|
||||
)
|
||||
if isinstance(value, (list, tuple))
|
||||
else ()
|
||||
)
|
||||
|
||||
|
||||
def validate_compactor_defaults(payload: Mapping[str, object]) -> None:
|
||||
if any(
|
||||
payload.get(key) is not None
|
||||
for key in ("context_management", "response_format", "stop", "stop_sequences", "tool_choice")
|
||||
):
|
||||
_reject(str(payload.get("model", "")), "Compactor defaults conflict with on-demand compaction")
|
||||
|
||||
|
||||
def _native_blocks(
|
||||
protocol: Literal["chat", "messages"], response: Mapping[str, object], model: str
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
if protocol == "messages":
|
||||
if response.get("stop_reason") != "compaction":
|
||||
_reject(model, "The native compactor did not finish compaction")
|
||||
return _objects(response.get("content"))
|
||||
choices: Final = _objects(response.get("choices"))
|
||||
choice: Final = choices[0] if len(choices) == 1 else _EMPTY
|
||||
message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY))
|
||||
provider_fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields", _EMPTY))
|
||||
return _objects(provider_fields.get("compaction_blocks"))
|
||||
|
||||
|
||||
def extract_summary(protocol: Literal["chat", "messages"], response: Mapping[str, object], model: str) -> str:
|
||||
blocks: Final = _native_blocks(protocol, response, model)
|
||||
if len(blocks) != 1:
|
||||
_reject(model, "Expected exactly one native compaction block")
|
||||
block: Final = _MAPPING.validate_python(blocks[0])
|
||||
content: Final = block.get("content")
|
||||
if (
|
||||
block.get("type") != "compaction"
|
||||
or not block.get("signature")
|
||||
or not isinstance(content, str)
|
||||
or not content.strip()
|
||||
):
|
||||
_reject(model, "The native compaction block is incomplete")
|
||||
return content
|
||||
|
||||
|
||||
async def dispatch(
|
||||
router: Router, protocol: Literal["chat", "messages"], payload: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
if protocol == "chat":
|
||||
chat_response: Final = await router.acompletion(
|
||||
model=str(payload["model"]),
|
||||
messages=_CHAT_MESSAGES.validate_python(payload["messages"]),
|
||||
stream=False,
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
|
||||
),
|
||||
)
|
||||
return _MAPPING.validate_python(chat_response.model_dump())
|
||||
return _MAPPING.validate_python(await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload))
|
||||
|
|
@ -36,6 +36,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
|
|||
return signature if isinstance(signature, str) else None
|
||||
|
||||
|
||||
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = _as_string_mapping(provider_specific_fields)
|
||||
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
return (
|
||||
tuple(
|
||||
block
|
||||
for raw_block in raw_blocks
|
||||
if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction"
|
||||
)
|
||||
if isinstance(raw_blocks, (list, tuple))
|
||||
else ()
|
||||
)
|
||||
|
||||
|
||||
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
|
||||
{"name", "type", "input_schema", "description", "cache_control", "strict"}
|
||||
)
|
||||
|
|
@ -1330,7 +1344,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_name_mapping: dict[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
new_content: Final[list[dict[str, Any]]] = []
|
||||
for choice in choices:
|
||||
for choice, compaction_blocks in (
|
||||
(choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))
|
||||
for choice in choices
|
||||
):
|
||||
new_content.extend(dict(block) for block in compaction_blocks)
|
||||
# Handle thinking blocks first
|
||||
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
|
||||
for thinking_block in choice.message.thinking_blocks:
|
||||
|
|
@ -1365,7 +1383,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
# Handle text content
|
||||
if choice.message.content is not None:
|
||||
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
|
||||
new_content.append(
|
||||
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
|
||||
)
|
||||
|
|
@ -1545,21 +1563,31 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
openai_finish_reason=openai_finish_reason
|
||||
)
|
||||
anthropic_finish_reason: Final = (
|
||||
"refusal"
|
||||
"compaction"
|
||||
if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction"
|
||||
else "refusal"
|
||||
if refusal_text is not None and translated_finish_reason != "max_tokens"
|
||||
else translated_finish_reason
|
||||
)
|
||||
# extract usage
|
||||
usage: Final[Usage] = getattr(response, "usage")
|
||||
anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
|
||||
|
||||
if polyfill_result is not None and polyfill_result.iterations_usage is not None:
|
||||
message_iteration: Final[UsageIteration] = {
|
||||
"type": "message",
|
||||
"input_tokens": anthropic_usage["input_tokens"],
|
||||
"output_tokens": usage.completion_tokens or 0,
|
||||
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
|
||||
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
|
||||
anthropic_usage: Final[AnthropicUsage] = (
|
||||
{
|
||||
**message_usage,
|
||||
"iterations": [
|
||||
*polyfill_iterations,
|
||||
UsageIteration(
|
||||
type="message",
|
||||
input_tokens=message_usage.get("input_tokens", 0),
|
||||
output_tokens=usage.completion_tokens or 0,
|
||||
),
|
||||
],
|
||||
}
|
||||
anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration]
|
||||
if polyfill_iterations is not None
|
||||
else message_usage
|
||||
)
|
||||
|
||||
translated_obj: Final = AnthropicMessagesResponse(
|
||||
id=response.id,
|
||||
|
|
|
|||
|
|
@ -580,7 +580,9 @@ def anthropic_messages_handler(
|
|||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
|
||||
if kwargs.get("compaction") is None and _should_route_to_responses_api(
|
||||
custom_llm_provider, original_model, model
|
||||
):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from ...common_utils import (
|
|||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
|
@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
*(["compaction"] if self._resolved_provider == "anthropic" else []),
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
@ -633,6 +635,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
beta_values.update(existing_beta)
|
||||
|
||||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
|
|
|
|||
|
|
@ -14487,6 +14487,7 @@
|
|||
"source": "https://docs.anthropic.com/en/docs/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 4e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -14525,6 +14526,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 6e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
|
|
@ -14737,6 +14739,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14774,6 +14777,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6-20260205": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14810,6 +14814,7 @@
|
|||
"prompt_cache_min_tokens": 4096
|
||||
},
|
||||
"claude-opus-4-7": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14848,6 +14853,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-7-20260416": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14885,6 +14891,7 @@
|
|||
"prompt_cache_min_tokens": 2048
|
||||
},
|
||||
"claude-fable-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -14924,6 +14931,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-fable-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -14964,6 +14972,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -15005,6 +15014,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-8": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -60324,6 +60334,7 @@
|
|||
"supports_audio_output": true
|
||||
},
|
||||
"claude-mythos-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -60363,6 +60374,7 @@
|
|||
}
|
||||
},
|
||||
"claude-mythos-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -60403,6 +60415,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-mythos-preview": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
|
|||
)
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import route_request
|
||||
from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails
|
||||
from litellm.router import Router
|
||||
|
|
@ -2533,7 +2534,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
user_model=user_model,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
llm_call_task: Final = asyncio.create_task(llm_call)
|
||||
llm_call_task: Final = asyncio.create_task(with_proxy_compaction_executor(llm_call, request))
|
||||
tasks.append(llm_call_task)
|
||||
|
||||
llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call
|
||||
|
|
|
|||
83
litellm/proxy/native_compaction.py
Normal file
83
litellm/proxy/native_compaction.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import asyncio
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from contextvars import Context
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
import httpx
|
||||
from fastapi import Request
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
compaction_executor,
|
||||
native_compaction_call,
|
||||
)
|
||||
|
||||
_ResultT: Final = TypeVar("_ResultT")
|
||||
_ASGI_APP: Final = TypeAdapter[ASGIApp](ASGIApp)
|
||||
_ROOT_PATH: Final = TypeAdapter(str)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
_REMOVED_HEADERS: Final = frozenset((b"content-length", b"x-litellm-call-id"))
|
||||
|
||||
|
||||
async def with_proxy_compaction_executor(call: Awaitable[_ResultT], request: Request) -> _ResultT:
|
||||
async def execute(protocol: Literal["chat", "messages"], payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
logging_disabled: Final = initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is True
|
||||
|
||||
async def dispatch() -> Mapping[str, object]:
|
||||
scope: Final = _JSON_OBJECT.validate_python(request.scope)
|
||||
root_path: Final = _ROOT_PATH.validate_python(scope.get("root_path", ""))
|
||||
path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages"
|
||||
url: Final = str(request.url.replace(path=root_path.rstrip("/") + path, query="", fragment=""))
|
||||
headers: Final = tuple(
|
||||
(name, value)
|
||||
for name, value in request.headers.raw
|
||||
if name.lower() not in _REMOVED_HEADERS
|
||||
and not (logging_disabled and name.decode("latin-1").lower() in UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS)
|
||||
)
|
||||
transport: Final = httpx.ASGITransport(
|
||||
app=_ASGI_APP.validate_python(scope["app"]),
|
||||
root_path=root_path,
|
||||
client=request.client, # pyright: ignore[reportArgumentType] # ASGI permits a missing client address
|
||||
raise_app_exceptions=False,
|
||||
)
|
||||
with native_compaction_call(), inherit_message_logging_privacy(logging_disabled):
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
follow_redirects=False,
|
||||
timeout=httpx.Timeout(None),
|
||||
) as client:
|
||||
async with client.stream(
|
||||
"POST", url, headers=headers, json=_JSON_OBJECT.validate_python(payload)
|
||||
) as response:
|
||||
if not response.is_success:
|
||||
raise BadRequestError(
|
||||
message=f"Native compaction child request failed (HTTP {response.status_code})",
|
||||
model="context_compaction",
|
||||
llm_provider="",
|
||||
)
|
||||
body: Final = await response.aread()
|
||||
try:
|
||||
return MappingProxyType(_JSON_OBJECT.validate_json(body))
|
||||
except ValidationError:
|
||||
raise BadRequestError(
|
||||
message="Native compaction child returned an invalid JSON object",
|
||||
model="context_compaction",
|
||||
llm_provider="",
|
||||
) from None
|
||||
|
||||
task: Final = Context().run(asyncio.create_task, dispatch())
|
||||
return await task
|
||||
|
||||
token: Final = compaction_executor.set(execute)
|
||||
try:
|
||||
return await call
|
||||
finally:
|
||||
compaction_executor.reset(token)
|
||||
|
|
@ -118,6 +118,14 @@ from litellm.llms.openai_like.model_info import (
|
|||
get_openai_compatible_model_info,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
arm_compaction,
|
||||
compact_to_fit,
|
||||
compaction_pending,
|
||||
initialize_compaction_state,
|
||||
is_native_compaction_call,
|
||||
reject_recursive_compactor,
|
||||
)
|
||||
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
|
||||
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
|
||||
|
|
@ -3634,6 +3642,7 @@ class Router:
|
|||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat")
|
||||
async with contextlib.AsyncExitStack() as deployment_slot:
|
||||
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
|
||||
deployment_slot.enter_context(max_parallel_requests_limit)
|
||||
|
|
@ -3642,7 +3651,7 @@ class Router:
|
|||
logging_obj=logging_obj,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
response = await litellm.acompletion(**input_kwargs)
|
||||
response = await litellm.acompletion(**compacted_input)
|
||||
|
||||
## CHECK CONTENT FILTER ERROR ##
|
||||
if isinstance(response, ModelResponse):
|
||||
|
|
@ -5240,8 +5249,14 @@ class Router:
|
|||
if custom_llm_provider is not None:
|
||||
response_kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
compacted_response_kwargs: Final = await compact_to_fit(
|
||||
self,
|
||||
deployment,
|
||||
response_kwargs,
|
||||
"messages" if getattr(original_generic_function, "__name__", "") == "anthropic_messages" else None,
|
||||
)
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await original_generic_function(**response_kwargs)
|
||||
response = await original_generic_function(**compacted_response_kwargs)
|
||||
|
||||
if self._should_raise_anthropic_refusal_error(
|
||||
model=model,
|
||||
|
|
@ -7394,6 +7409,11 @@ class Router:
|
|||
If it fails after num_retries, fall back to another model group
|
||||
"""
|
||||
model_group: Final[str | None] = kwargs.get("model")
|
||||
if (
|
||||
kwargs.get("original_function") == self._acompletion
|
||||
or getattr(kwargs.get("original_generic_function"), "__name__", "") == "anthropic_messages"
|
||||
):
|
||||
kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs)
|
||||
clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary
|
||||
if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets):
|
||||
_fallback_metadata_key: Final = _get_router_metadata_variable_name(
|
||||
|
|
@ -12089,8 +12109,8 @@ class Router:
|
|||
|
||||
def _count_pre_call_check_tokens(
|
||||
self,
|
||||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
messages: Sequence[Mapping[str, object]] | None,
|
||||
input: str | list[object] | None,
|
||||
request_kwargs: Mapping[str, object] | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
|
|
@ -12237,7 +12257,9 @@ class Router:
|
|||
_rate_limit_error = False
|
||||
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs)
|
||||
|
||||
has_countable_input: Final = messages is not None or input is not None
|
||||
has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending(
|
||||
request_kwargs
|
||||
)
|
||||
|
||||
## get model group RPM ##
|
||||
dt: Final = get_utc_datetime()
|
||||
|
|
@ -13436,6 +13458,8 @@ class Router:
|
|||
registered_model_name: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> str:
|
||||
if is_native_compaction_call():
|
||||
return registered_model_name
|
||||
if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)):
|
||||
return registered_model_name
|
||||
cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs)
|
||||
|
|
@ -13514,6 +13538,7 @@ class Router:
|
|||
model=registered_model_name, request_kwargs=request_kwargs
|
||||
)
|
||||
if selected_strategy is None:
|
||||
arm_compaction(request_kwargs, None)
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
self._stamp_or_clear_metadata_key(
|
||||
request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
|
||||
|
|
@ -13524,6 +13549,24 @@ class Router:
|
|||
return None
|
||||
|
||||
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
|
||||
reject_recursive_compactor(registered_model_name)
|
||||
arm_compaction(
|
||||
request_kwargs,
|
||||
selected_strategy.strategy.config.context_compaction
|
||||
if isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
else None,
|
||||
tuple(
|
||||
dict.fromkeys(
|
||||
member
|
||||
for pool in selected_strategy.strategy.config.tiers.values()
|
||||
for member in ((pool,) if isinstance(pool, str) else pool)
|
||||
)
|
||||
)
|
||||
if isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
else (),
|
||||
)
|
||||
|
||||
await authorize_member_auto_router_inference(
|
||||
deployment=self._selected_strategy_marker_deployment(
|
||||
|
|
|
|||
|
|
@ -68,6 +68,68 @@ still resolve to a deployment in `model_list`; this configuration does not creat
|
|||
- abc
|
||||
```
|
||||
|
||||
### Context compaction
|
||||
|
||||
Native compact-to-fit is enabled by default for Complexity Router Chat Completions
|
||||
and Anthropic Messages requests. Near the selected deployment's context limit,
|
||||
it compacts older history while keeping that answering deployment pinned.
|
||||
It takes precedence over context-window escalation on these request types;
|
||||
other API surfaces retain their existing behavior
|
||||
|
||||
```yaml
|
||||
complexity_router_config:
|
||||
context_compaction:
|
||||
model: native-compactor
|
||||
max_tokens: 4096
|
||||
trigger_ratio: 0.9
|
||||
timeout_seconds: 120
|
||||
```
|
||||
|
||||
The three numeric values shown are defaults. With no `model` override, the router
|
||||
uses the first configured tier group whose deployments all support Anthropic native
|
||||
on-demand compaction and fit the old history plus output allowance. Selection follows
|
||||
the configured tier order and never searches unrelated gateway deployments. If no
|
||||
group qualifies, the request fails before compaction. A fitting request needs no compactor
|
||||
|
||||
Compactor capability comes from `supports_anthropic_compaction` in the model catalog,
|
||||
read through `get_model_info`, rather than a model-name allowlist. This flag specifically
|
||||
means support for Anthropic's signed on-demand compaction operation. Catalog updates
|
||||
or `litellm.register_model` can enable a compatible new compactor without router changes
|
||||
|
||||
OpenAI-compatible gateways receive the native operation and its required beta header.
|
||||
For Messages calls without native Messages passthrough, compaction uses the Chat bridge
|
||||
automatically; ordinary OpenAI/Azure Messages routing keeps its existing behavior
|
||||
|
||||
Set `model` to override automatic selection with a configured regular model group,
|
||||
such as one backed by `anthropic/claude-sonnet-5`. This also supports custom gateway
|
||||
aliases or newly supported native models not yet recognized automatically. Another
|
||||
auto-router cannot be the compactor. Set `context_compaction: false` to disable this
|
||||
behavior; an omitted setting or empty object enables it. Explicit null is normalized
|
||||
to false so the opt-out survives management API serialization. Escalation remains a separate setting
|
||||
|
||||
Anthropic generates the summary using its native compaction operation. The router
|
||||
replays that summary as ordinary text to the same selected answering deployment,
|
||||
for example `anthropic/claude-haiku-4-5-20251001`. It does not replay the signed
|
||||
compaction block to Haiku. Answering models need no native-compaction capability:
|
||||
DeepSeek and other Chat/Messages targets receive the same ordinary-text summary.
|
||||
Using a model inside a coding agent does not imply that its provider API supports
|
||||
native compaction; the agent may manage summarization itself. Leading system/developer instructions, the Messages
|
||||
system prompt, and the latest user turn with its following tool calls/results
|
||||
stay unchanged. The history split must preserve complete tool pairs
|
||||
|
||||
The input budget is `floor(trigger_ratio * max_input_tokens) - output_reserve`.
|
||||
Token counts are estimates; the reserve uses the request's output limit or the
|
||||
model's known output limit. The compacted request is checked again before dispatch
|
||||
|
||||
Upgrading enables lossy history summarization and an extra paid call near the limit
|
||||
unless explicitly disabled. Each request gets at most one billable compaction attempt. On the proxy, the caller
|
||||
must have access to the compactor and sufficient budget. Compactor retries and
|
||||
fallbacks are disabled. Automatic mode leaves unknown input limits to the provider;
|
||||
when the output reserve is unknown or leaves no usable input budget, it still rejects
|
||||
inputs exceeding the known window. An explicit compactor requires a positive reserved-input budget. Detected overflow, unsafe history,
|
||||
or a summary that still does not fit fails closed. There is no chunking, prompt-based summarization,
|
||||
or context-window escalation fallback
|
||||
|
||||
### Capability forecasting
|
||||
|
||||
Set `classifier_type: capability` to use
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
|
|||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
|
||||
from litellm.router_strategy.complexity_router.tier_predictor import (
|
||||
TierSuccessPredictor,
|
||||
resolve_tier_artifact,
|
||||
|
|
@ -3241,7 +3242,11 @@ class ComplexityRouter(CustomLogger):
|
|||
resolved_messages: Sequence[Mapping[str, object]] | None,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> _RequestContextFit:
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
if (
|
||||
compaction_pending(request_kwargs)
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
|
||||
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
|
||||
(self.config.default_model,) if self.config.default_model else ()
|
||||
|
|
@ -3267,7 +3272,11 @@ class ComplexityRouter(CustomLogger):
|
|||
(the placement stands). Only a real tokenizer count ever moves a request, escalation
|
||||
lands only on groups whose every deployment declares a fitting window, and a group
|
||||
with no resolvable window is never moved on faith in either direction."""
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
if (
|
||||
compaction_pending(request_kwargs)
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
return None
|
||||
pools: Final = self._tier_pools()
|
||||
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))
|
||||
|
|
|
|||
|
|
@ -833,6 +833,15 @@ class CustomDimension(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ContextCompactionConfig(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
model: str | None = Field(default=None, min_length=1)
|
||||
trigger_ratio: float = Field(default=0.9, gt=0, lt=1)
|
||||
max_tokens: int = Field(default=4096, ge=512)
|
||||
timeout_seconds: float = Field(default=120, gt=0)
|
||||
|
||||
|
||||
class ComplexityRouterConfig(BaseModel):
|
||||
"""Configuration for the ComplexityRouter."""
|
||||
|
||||
|
|
@ -1304,6 +1313,16 @@ class ComplexityRouterConfig(BaseModel):
|
|||
),
|
||||
)
|
||||
|
||||
context_compaction: ContextCompactionConfig | Literal[False] | None = Field(
|
||||
default_factory=ContextCompactionConfig,
|
||||
description="Native Anthropic compact-to-fit is enabled by default for Chat and Messages. Selects a compatible configured tier model unless model is specified. Set false to disable (null is normalized to false). Takes precedence over context-window escalation on supported requests.",
|
||||
)
|
||||
|
||||
@field_validator("context_compaction", mode="before")
|
||||
@classmethod
|
||||
def _normalize_context_compaction(cls, value: object) -> object:
|
||||
return False if value is None else value
|
||||
|
||||
enable_context_window_escalation: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
|
|
|
|||
348
litellm/router_strategy/complexity_router/context_compaction.py
Normal file
348
litellm/router_strategy/complexity_router/context_compaction.py
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, NoReturn, TypeAlias
|
||||
from typing import Protocol as CallableProtocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
parent_session_kwargs,
|
||||
sanitized_forwardable_call_metadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # legacy owner receives validated call details
|
||||
)
|
||||
from litellm.llms.anthropic import compaction as anthropic_compaction
|
||||
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
Protocol: TypeAlias = Literal["chat", "messages"]
|
||||
|
||||
|
||||
class CompactionExecutor(CallableProtocol):
|
||||
async def __call__(self, protocol: Protocol, payload: Mapping[str, object]) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
compaction_executor: Final[ContextVar[CompactionExecutor | None]] = ContextVar("compaction_executor", default=None)
|
||||
_native_child: Final[ContextVar[bool]] = ContextVar("native_compaction_child", default=False)
|
||||
_STATE_KEY: Final = "_context_compaction_state"
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
|
||||
_DICT: Final = TypeAdapter(dict[str, object])
|
||||
_INPUT: Final[TypeAdapter[str | list[object] | None]] = TypeAdapter(
|
||||
str | list[object] | None
|
||||
) # mutable-ok: Responses normalization requires a JSON list
|
||||
_SEQUENCE: Final = TypeAdapter(tuple[object, ...])
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
@dataclass(slots=True, repr=False)
|
||||
class CompactionState:
|
||||
config: ContextCompactionConfig | None = None
|
||||
candidates: tuple[str, ...] = ()
|
||||
summary: tuple[str, asyncio.Task[str]] | None = field(default=None, repr=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InputBudget:
|
||||
window: int | None
|
||||
available: int | None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def native_compaction_call() -> Generator[None]:
|
||||
token: Final = _native_child.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_native_child.reset(token)
|
||||
|
||||
|
||||
def initialize_compaction_state(kwargs: Mapping[str, object]) -> CompactionState:
|
||||
existing: Final = kwargs.get(_STATE_KEY)
|
||||
return existing if isinstance(existing, CompactionState) else CompactionState()
|
||||
|
||||
|
||||
def arm_compaction(
|
||||
kwargs: Mapping[str, object],
|
||||
config: ContextCompactionConfig | Literal[False] | None,
|
||||
candidates: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
state: Final = kwargs.get(_STATE_KEY)
|
||||
if isinstance(state, CompactionState):
|
||||
state.config = config if isinstance(config, ContextCompactionConfig) else None
|
||||
state.candidates = candidates
|
||||
|
||||
|
||||
def reject_recursive_compactor(model: str) -> None:
|
||||
if _native_child.get():
|
||||
_reject(model, "The compactor must be a regular model group, not an auto-router")
|
||||
|
||||
|
||||
def is_native_compaction_call() -> bool:
|
||||
return _native_child.get()
|
||||
|
||||
|
||||
def compaction_pending(kwargs: Mapping[str, object] | None) -> bool:
|
||||
state: Final = kwargs.get(_STATE_KEY) if kwargs is not None else None
|
||||
return isinstance(state, CompactionState) and state.config is not None
|
||||
|
||||
|
||||
def _reject(model: str, reason: str) -> NoReturn:
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="")
|
||||
|
||||
|
||||
def _positive_int(value: object) -> int | None:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
|
||||
|
||||
|
||||
def _objects(value: object) -> tuple[Mapping[str, object], ...]:
|
||||
return (
|
||||
tuple(
|
||||
_MAPPING.validate_python(block) for block in _SEQUENCE.validate_python(value) if isinstance(block, Mapping)
|
||||
)
|
||||
if isinstance(value, (list, tuple))
|
||||
else ()
|
||||
)
|
||||
|
||||
|
||||
def _blocks(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
return _objects(message.get("content"))
|
||||
|
||||
|
||||
def _tool_ids(messages: Sequence[Mapping[str, object]], *, results: bool) -> frozenset[str]:
|
||||
return frozenset(
|
||||
str(identifier)
|
||||
for message in messages
|
||||
for identifier in (
|
||||
*((message.get("tool_call_id"),) if results and message.get("role") == "tool" else ()),
|
||||
*(
|
||||
block.get("tool_use_id" if results else "id")
|
||||
for block in _blocks(message)
|
||||
if block.get("type") == ("tool_result" if results else "tool_use")
|
||||
),
|
||||
*(call.get("id") for call in _objects(message.get("tool_calls")) if not results),
|
||||
)
|
||||
if identifier is not None
|
||||
)
|
||||
|
||||
|
||||
def _history(
|
||||
messages: Sequence[Mapping[str, object]], model: str
|
||||
) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]:
|
||||
instructions: Final = tuple(takewhile(lambda message: message.get("role") in ("system", "developer"), messages))
|
||||
conversation: Final = tuple(messages[len(instructions) :])
|
||||
if any(message.get("role") in ("system", "developer") for message in conversation):
|
||||
_reject(model, "Mid-conversation instructions cannot be compacted")
|
||||
split: Final = next(
|
||||
(
|
||||
index
|
||||
for index in range(len(conversation) - 1, -1, -1)
|
||||
if conversation[index].get("role") == "user"
|
||||
and not any(block.get("type") == "tool_result" for block in _blocks(conversation[index]))
|
||||
),
|
||||
0,
|
||||
)
|
||||
prefix: Final = conversation[:split]
|
||||
if not prefix or _tool_ids(prefix, results=False) != _tool_ids(prefix, results=True):
|
||||
_reject(model, "No closed older conversation is available to compact without changing the latest request")
|
||||
return instructions, prefix, conversation[split:]
|
||||
|
||||
|
||||
async def _count(router: Router, payload: Mapping[str, object]) -> int:
|
||||
return await asyncio.to_thread(
|
||||
router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # Reuse Router's shared admission counter
|
||||
messages=_MESSAGES.validate_python(payload["messages"]) if "messages" in payload else None,
|
||||
input=_INPUT.validate_python(payload.get("input")),
|
||||
request_kwargs=payload,
|
||||
)
|
||||
|
||||
|
||||
def _budget(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
|
||||
) -> InputBudget:
|
||||
model: Final = str(payload.get("model", ""))
|
||||
info: Final = _MAPPING.validate_python(
|
||||
router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
|
||||
)
|
||||
window_value: Final = info.get("max_input_tokens")
|
||||
window: Final = window_value if isinstance(window_value, int) else None
|
||||
output_value: Final = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("max_completion_tokens", "max_output_tokens", "max_tokens")
|
||||
if payload.get(key) is not None
|
||||
),
|
||||
info.get("max_output_tokens"),
|
||||
)
|
||||
output: Final = _positive_int(output_value)
|
||||
if output_value is not None and output is None:
|
||||
_reject(model, "The output allowance must be a positive integer")
|
||||
return InputBudget(window, int(window * ratio) - output if window is not None and output is not None else None)
|
||||
|
||||
|
||||
async def _compactor_model(router: Router, state: CompactionState, payload: Mapping[str, object]) -> str:
|
||||
config: Final = state.config
|
||||
if config is not None and config.model is not None:
|
||||
return config.model
|
||||
needed: Final = await _count(router, payload)
|
||||
for candidate in state.candidates:
|
||||
deployments: Final = tuple(router.get_model_list(model_name=candidate) or ())
|
||||
if deployments and all(
|
||||
anthropic_compaction.supports_native_compaction(_MAPPING.validate_python(deployment["litellm_params"]))
|
||||
and (budget := _budget(router, deployment, payload, 0.9)).available is not None
|
||||
and needed <= budget.available
|
||||
for deployment in deployments
|
||||
):
|
||||
return candidate
|
||||
return _reject(
|
||||
str(payload.get("model", "")),
|
||||
"No configured tier model supports native compaction with enough context; set context_compaction.model",
|
||||
)
|
||||
|
||||
|
||||
async def _generate_summary(
|
||||
router: Router, protocol: Protocol, payload: Mapping[str, object], timeout_seconds: float
|
||||
) -> str:
|
||||
executor: Final = compaction_executor.get()
|
||||
with native_compaction_call():
|
||||
response: Final = await asyncio.wait_for(
|
||||
executor(protocol, payload)
|
||||
if executor is not None
|
||||
else anthropic_compaction.dispatch(router, protocol, payload),
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
return anthropic_compaction.extract_summary(protocol, response, str(payload["model"]))
|
||||
|
||||
|
||||
async def compact_to_fit(
|
||||
router: Router,
|
||||
deployment: Mapping[str, object],
|
||||
payload: Mapping[str, object],
|
||||
protocol: Protocol | None,
|
||||
) -> Mapping[str, object]:
|
||||
state: Final = payload.get(_STATE_KEY)
|
||||
config: Final = state.config if isinstance(state, CompactionState) else None
|
||||
if config is None and not _native_child.get():
|
||||
return payload
|
||||
model: Final = str(payload.get("model", ""))
|
||||
limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9)
|
||||
budget: Final = limits.available
|
||||
if budget is None or budget <= 0:
|
||||
if (
|
||||
config is not None
|
||||
and config.model is None
|
||||
and (limits.window is None or await _count(router, payload) <= limits.window)
|
||||
):
|
||||
return payload
|
||||
_reject(model, "A known input window and a smaller output allowance are required")
|
||||
if _native_child.get():
|
||||
anthropic_compaction.validate_compactor_defaults(payload)
|
||||
if await _count(router, payload) > budget:
|
||||
_reject(model, "The selected compactor cannot fit the history and output allowance")
|
||||
return payload
|
||||
if await _count(router, payload) <= budget:
|
||||
return payload
|
||||
if protocol is None or config is None or not isinstance(state, CompactionState):
|
||||
_reject(model, "Only Chat Completions and Anthropic Messages support compact-to-fit")
|
||||
if any(payload.get(key) is not None for key in anthropic_compaction.CLIENT_COMPACTION_PARAMS):
|
||||
_reject(model, "Do not combine compact-to-fit with client-managed compaction")
|
||||
messages: Final = _MESSAGES.validate_python(payload["messages"])
|
||||
instructions, prefix, tail = _history(messages, model)
|
||||
if any(
|
||||
message.get("thinking_blocks")
|
||||
or any(block.get("type") in ("thinking", "redacted_thinking") for block in _blocks(message))
|
||||
for message in tail
|
||||
):
|
||||
_reject(model, "Retained native reasoning cannot be replayed after text compaction")
|
||||
retained: Final = MappingProxyType({**payload, "messages": _MESSAGES.validate_python((*instructions, *tail))})
|
||||
if await _count(router, retained) >= budget:
|
||||
_reject(model, "The retained instructions, tools and latest turn leave no room for a summary")
|
||||
parent_metadata: Final = _MAPPING.validate_python(
|
||||
payload.get("litellm_metadata") or payload.get("metadata") or _EMPTY
|
||||
)
|
||||
forwarded_metadata: Final = sanitized_forwardable_call_metadata(parent_metadata, "autorouter_compaction")
|
||||
logging_disabled: Final = should_redact_message_logging(
|
||||
_DICT.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
"litellm_params": payload,
|
||||
"standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(
|
||||
_DICT.validate_python(payload)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
metadata: Final = _DICT.validate_python(
|
||||
MappingProxyType({key: value for key, value in forwarded_metadata.items() if key != "user_api_key_auth"})
|
||||
if compaction_executor.get() is not None
|
||||
else forwarded_metadata
|
||||
)
|
||||
native_request: Final = MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"messages": _MESSAGES.validate_python((*instructions, *prefix)),
|
||||
**anthropic_compaction.request_kwargs(),
|
||||
"max_tokens": config.max_tokens,
|
||||
"stream": False,
|
||||
"num_retries": 0,
|
||||
"disable_fallbacks": True,
|
||||
"timeout": config.timeout_seconds,
|
||||
"litellm_metadata" if protocol == "messages" else "metadata": metadata,
|
||||
**parent_session_kwargs(payload),
|
||||
**MappingProxyType({key: payload[key] for key in ("system", "tools", "user") if key in payload}),
|
||||
}
|
||||
)
|
||||
compactor: Final = await _compactor_model(router, state, native_request)
|
||||
native_payload: Final = MappingProxyType({**native_request, "model": compactor})
|
||||
identity: Final = hashlib.sha256(
|
||||
json.dumps(
|
||||
(
|
||||
protocol,
|
||||
compactor,
|
||||
native_payload["messages"],
|
||||
native_payload.get("system"),
|
||||
native_payload.get("tools"),
|
||||
),
|
||||
sort_keys=True,
|
||||
).encode()
|
||||
).hexdigest()
|
||||
if state.summary is None:
|
||||
with inherit_message_logging_privacy(logging_disabled):
|
||||
state.summary = (
|
||||
identity,
|
||||
asyncio.create_task(_generate_summary(router, protocol, native_payload, config.timeout_seconds)),
|
||||
)
|
||||
if state.summary[0] != identity:
|
||||
_reject(model, "The history changed after this request's single compaction attempt")
|
||||
summary: Final = await state.summary[1]
|
||||
summary_message: Final = MappingProxyType(
|
||||
{"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary}
|
||||
)
|
||||
compacted: Final = MappingProxyType(
|
||||
{
|
||||
**payload,
|
||||
"messages": _MESSAGES.validate_python((*instructions, summary_message, *tail)),
|
||||
}
|
||||
)
|
||||
if await _count(router, compacted) > budget:
|
||||
_reject(model, "The native summary and retained conversation still exceed the selected deployment's budget")
|
||||
return compacted
|
||||
|
|
@ -24,7 +24,7 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
|
|||
|
||||
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
|
||||
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding"]
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "compactor"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -154,6 +154,7 @@ def strategy_router_dependencies(
|
|||
dict.fromkeys(
|
||||
tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier"))
|
||||
+ _named(litellm_params.get("complexity_router_default_model"), "default")
|
||||
+ _named(_mapping(complexity.get("context_compaction")).get("model"), "compactor")
|
||||
+ (
|
||||
_named(classifier.get("model"), "classifier")
|
||||
if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES
|
||||
|
|
|
|||
|
|
@ -216,11 +216,20 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False):
|
|||
caller: ToolCaller | None
|
||||
|
||||
|
||||
class CompactionBlock(TypedDict, total=False):
|
||||
"""Native compaction block, signed for on-demand compaction."""
|
||||
|
||||
type: Required[ReadOnly[Literal["compaction"]]]
|
||||
content: ReadOnly[str | None]
|
||||
signature: ReadOnly[str]
|
||||
|
||||
|
||||
AnthropicMessagesAssistantMessageValues = (
|
||||
AnthropicMessagesTextParam
|
||||
| AnthropicMessagesToolUseParam
|
||||
| ChatCompletionThinkingBlock
|
||||
| ChatCompletionRedactedThinkingBlock
|
||||
| CompactionBlock
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -390,6 +399,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = (
|
|||
)
|
||||
|
||||
|
||||
class AnthropicCompaction(TypedDict, total=False):
|
||||
type: Required[ReadOnly[Literal["summarize"]]]
|
||||
instructions: ReadOnly[str]
|
||||
|
||||
|
||||
class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
||||
max_tokens: int | None
|
||||
metadata: AnthropicMetadata | dict | None
|
||||
|
|
@ -405,6 +419,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
|||
top_p: float | None
|
||||
mcp_servers: list[AnthropicMcpServerTool] | None
|
||||
context_management: dict[str, Any] | None
|
||||
compaction: ReadOnly[AnthropicCompaction | None]
|
||||
container: dict[str, Any] | None # Container config with skills for code execution
|
||||
output_format: AnthropicOutputSchema | None # Structured outputs support
|
||||
speed: str | None # Fast mode support for Opus models
|
||||
|
|
@ -564,13 +579,6 @@ class ContextManagementResponse(TypedDict, total=False):
|
|||
applied_edits: list[AppliedEdit]
|
||||
|
||||
|
||||
class CompactionBlock(TypedDict, total=False):
|
||||
"""Synthesized ``compaction`` content block (compact_20260112)."""
|
||||
|
||||
type: Required[Literal["compaction"]]
|
||||
content: str | None
|
||||
|
||||
|
||||
class UsageIteration(TypedDict, total=False):
|
||||
"""One sampling iteration's token usage (compact_20260112)."""
|
||||
|
||||
|
|
@ -743,6 +751,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
WEB_SEARCH_2025_03_05 = "web-search-2025-03-05"
|
||||
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
|
||||
COMPACT_2026_01_12 = "compact-2026-01-12"
|
||||
COMPACT_2026_09_04 = "compact-2026-09-04"
|
||||
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
|
||||
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
|
||||
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Sequence
|
||||
from typing import Any, Literal, TypeAlias
|
||||
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
|
@ -6,8 +7,10 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicResponseContentBlockText,
|
||||
AnthropicResponseContentBlockToolUse,
|
||||
AnthropicStopDetails,
|
||||
CompactionBlock,
|
||||
ContextManagementResponse,
|
||||
ServerToolUsage,
|
||||
UsageIteration,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -56,6 +59,7 @@ AnthropicResponseContentBlock: TypeAlias = (
|
|||
| AnthropicResponseToolUseBlock
|
||||
| AnthropicResponseThinkingBlock
|
||||
| AnthropicResponseRedactedThinkingBlock
|
||||
| CompactionBlock
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -66,6 +70,7 @@ class AnthropicUsage(TypedDict, total=False):
|
|||
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
iterations: ReadOnly[Sequence[UsageIteration]]
|
||||
|
||||
"""
|
||||
Cache Tokens Used
|
||||
|
|
@ -91,7 +96,9 @@ class AnthropicMessagesResponse(TypedDict, total=False):
|
|||
id: str
|
||||
model: str | None # This represents the Model type from Anthropic
|
||||
role: Literal["assistant"] | None
|
||||
stop_reason: Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] | None
|
||||
stop_reason: ReadOnly[
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
|
||||
]
|
||||
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
|
||||
stop_sequence: str | None
|
||||
type: Literal["message"] | None
|
||||
|
|
|
|||
|
|
@ -172,6 +172,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_output_config: bool | None
|
||||
supports_image_size: bool | None
|
||||
supports_anthropic_thinking_payload: ReadOnly[bool | None]
|
||||
supports_anthropic_compaction: ReadOnly[bool | None]
|
||||
supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None]
|
||||
vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None]
|
||||
bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None
|
||||
|
|
@ -2959,6 +2960,7 @@ RoutingDecisionCause = Literal[
|
|||
|
||||
InternalCallOrigin = Literal[
|
||||
"autorouter_classifier",
|
||||
"autorouter_compaction",
|
||||
"shadow_eval_router",
|
||||
"shadow_eval_judge",
|
||||
"llm_as_a_judge_guardrail",
|
||||
|
|
@ -3818,6 +3820,7 @@ all_litellm_params = (
|
|||
agentic_loop_internal_litellm_params
|
||||
+ [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params]
|
||||
+ [
|
||||
"_context_compaction_state",
|
||||
"metadata",
|
||||
"litellm_metadata",
|
||||
"keepalive_seconds",
|
||||
|
|
|
|||
|
|
@ -6109,6 +6109,7 @@ def _get_model_info_helper(
|
|||
supports_tool_search=_model_info.get("supports_tool_search", None),
|
||||
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
|
||||
supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None),
|
||||
supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None),
|
||||
supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),
|
||||
supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None),
|
||||
supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),
|
||||
|
|
|
|||
|
|
@ -14487,6 +14487,7 @@
|
|||
"source": "https://docs.anthropic.com/en/docs/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 4e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -14525,6 +14526,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 6e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
|
|
@ -14737,6 +14739,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14774,6 +14777,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6-20260205": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14810,6 +14814,7 @@
|
|||
"prompt_cache_min_tokens": 4096
|
||||
},
|
||||
"claude-opus-4-7": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14848,6 +14853,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-7-20260416": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14885,6 +14891,7 @@
|
|||
"prompt_cache_min_tokens": 2048
|
||||
},
|
||||
"claude-fable-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -14924,6 +14931,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-fable-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -14964,6 +14972,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -15005,6 +15014,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-8": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -60324,6 +60334,7 @@
|
|||
"supports_audio_output": true
|
||||
},
|
||||
"claude-mythos-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -60363,6 +60374,7 @@
|
|||
}
|
||||
},
|
||||
"claude-mythos-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -60403,6 +60415,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-mythos-preview": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
|
|||
|
|
@ -815,6 +815,9 @@
|
|||
"supports_adaptive_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_anthropic_compaction": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_anthropic_thinking_payload": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ ALLOWED_FILES = [
|
|||
"../../litellm/llms/custom_httpx/httpx_handler.py",
|
||||
"../../litellm/llms/openai/common_utils.py",
|
||||
"../../litellm/experimental_mcp_client/client.py",
|
||||
"../../litellm/proxy/native_compaction.py",
|
||||
# when running on ci/cd
|
||||
"./litellm/__init__.py",
|
||||
"./litellm/llms/custom_httpx/http_handler.py",
|
||||
|
|
@ -22,6 +23,7 @@ ALLOWED_FILES = [
|
|||
"./litellm/llms/custom_httpx/httpx_handler.py",
|
||||
"./litellm/llms/openai/common_utils.py",
|
||||
"./litellm/experimental_mcp_client/client.py",
|
||||
"./litellm/proxy/native_compaction.py",
|
||||
]
|
||||
|
||||
warning_msg = "this is a serious violation that can impact latency. Creating Async clients per request can add +500ms per request"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
iter_client_callback_metadata_dicts,
|
||||
)
|
||||
|
|
@ -189,6 +194,18 @@ def test_empty_kwargs_returns_empty_params():
|
|||
assert dict(params) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("child_privacy", (False, True))
|
||||
def test_inherited_privacy_only_strengthens_child_and_resets(child_privacy: bool) -> None:
|
||||
kwargs: Final = TypeAdapter(dict[str, bool]).validate_python(MappingProxyType({"turn_off_message_logging": child_privacy}))
|
||||
with inherit_message_logging_privacy(False):
|
||||
assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy
|
||||
with inherit_message_logging_privacy(True), inherit_message_logging_privacy(False):
|
||||
params: Final = initialize_standard_callback_dynamic_params(kwargs)
|
||||
assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy
|
||||
assert params["turn_off_message_logging"] is True
|
||||
assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None
|
||||
|
||||
|
||||
def test_newrelic_callback_params_are_not_extracted_from_request_kwargs():
|
||||
kwargs = {
|
||||
"newrelic_api_key": "caller-key",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
|
||||
import pytest
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
|
|
@ -17,6 +20,7 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
|||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.utils import ServerToolUse, Usage
|
||||
|
||||
|
|
@ -3761,6 +3765,64 @@ def test_multiple_compaction_blocks():
|
|||
assert compaction_blocks[1]["content"] == "Second summary..."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("messages_api", [False, True])
|
||||
async def test_native_compaction_wire_roundtrip(messages_api: bool, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
|
||||
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
|
||||
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
|
||||
operation: Final = {"type": "summarize", "instructions": "Keep identifiers"}
|
||||
usage: Final = {
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}],
|
||||
}
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert len(request.headers.get_list("anthropic-beta")) == 1
|
||||
assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == {
|
||||
"compact-2026-09-04",
|
||||
"interleaved-thinking-2025-05-14",
|
||||
}
|
||||
if "compaction" in payload:
|
||||
assert payload["compaction"] == operation
|
||||
else:
|
||||
assert payload["messages"][0] == {"role": "assistant", "content": [block]}
|
||||
body: Final = dict(
|
||||
id="msg_compact",
|
||||
type="message",
|
||||
role="assistant",
|
||||
model="claude-sonnet-5",
|
||||
content=[block],
|
||||
stop_reason="compaction",
|
||||
usage=usage,
|
||||
)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as transport:
|
||||
client: Final = AsyncHTTPHandler()
|
||||
await client.client.aclose()
|
||||
client.client = transport
|
||||
call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion
|
||||
params: Final = dict(
|
||||
model="anthropic/claude-sonnet-5",
|
||||
api_key="test",
|
||||
client=client,
|
||||
max_tokens=512,
|
||||
extra_headers={"Anthropic-Beta": "interleaved-thinking-2025-05-14"},
|
||||
)
|
||||
response: Final = await call(
|
||||
messages=[{"role": "user", "content": "Remember identifiers"}], compaction=operation, **params
|
||||
)
|
||||
message: Final = response if messages_api else response.choices[0].message.model_dump()
|
||||
blocks: Final = message["content"] if messages_api else message["provider_specific_fields"]["compaction_blocks"]
|
||||
assert blocks == [block]
|
||||
if messages_api:
|
||||
assert response["usage"] == usage
|
||||
replay: Final = {"role": "assistant", "content": blocks} if messages_api else message
|
||||
await call(messages=[replay, {"role": "user", "content": "Continue"}], **params)
|
||||
|
||||
|
||||
def test_compaction_block_request_transformation():
|
||||
"""
|
||||
Test that compaction blocks from provider_specific_fields are correctly
|
||||
|
|
|
|||
|
|
@ -60,6 +60,27 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None:
|
|||
assert result["usage"]["input_tokens"] == 10
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,count,expected_stop",
|
||||
[
|
||||
("", 1, "compaction"),
|
||||
(None, 1, "compaction"),
|
||||
("Answer", 1, "max_tokens"),
|
||||
(" ", 1, "max_tokens"),
|
||||
("", 2, "max_tokens"),
|
||||
("", 0, "max_tokens"),
|
||||
],
|
||||
)
|
||||
def test_native_compaction_response_roundtrip(text: str | None, count: int, expected_stop: str) -> None:
|
||||
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
|
||||
message: Final = Message(content=text, provider_specific_fields={"compaction_blocks": [block] * count})
|
||||
response: Final = ModelResponse(choices=[Choices(message=message, finish_reason="length")], usage=Usage())
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
|
||||
expected_text: Final = [{"type": "text", "text": text}] if text is not None and (text != "" or not count) else []
|
||||
assert result["content"] == [*([block] * count), *expected_text]
|
||||
assert result["stop_reason"] == expected_stop
|
||||
|
||||
|
||||
def test_translate_chat_refusal_to_anthropic_response():
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-refusal",
|
||||
|
|
|
|||
175
tests/test_litellm/proxy/test_native_compaction.py
Normal file
175
tests/test_litellm/proxy/test_native_compaction.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import asyncio
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, Request
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.proxy import common_request_processing, proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash, get_request_stash
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_executor, reject_recursive_compactor
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
||||
async def _child(protocol: Literal["chat", "messages"] = "chat") -> Mapping[str, object]:
|
||||
executor: Final = compaction_executor.get()
|
||||
assert executor is not None
|
||||
metadata: Final = TypeAdapter(dict[str, str]).validate_json(b'{"user_api_key_user_id":"fixture-user"}')
|
||||
return await executor(protocol, MappingProxyType({
|
||||
"model": "compactor", "metadata" if protocol == "chat" else "litellm_metadata": metadata,
|
||||
}))
|
||||
|
||||
|
||||
def _request(app: FastAPI, headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request:
|
||||
return Request(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({
|
||||
"type": "http", "app": app, "scheme": "https", "server": ("proxy.test", 443),
|
||||
"path": "/gateway/parent", "root_path": "/gateway", "query_string": b"",
|
||||
"client": ("192.0.2.1", 4321), "headers": headers,
|
||||
})))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("chat", "messages"))
|
||||
async def test_child_preserves_credentials_and_isolates_context(protocol: Literal["chat", "messages"]) -> None:
|
||||
app: Final = FastAPI()
|
||||
stash: Final = get_or_create_request_stash()
|
||||
path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages"
|
||||
|
||||
@app.post(path)
|
||||
async def endpoint(request: Request) -> Mapping[str, object]:
|
||||
assert get_request_stash() is None and compaction_executor.get() is None
|
||||
assert request.client == ("192.0.2.1", 4321) and request.url.scheme == "https"
|
||||
assert request.scope["root_path"] == "/gateway" and request.cookies["session"] == "fixture"
|
||||
assert request.headers["authorization"] == "Bearer sk-compaction-fixture"
|
||||
assert "x-litellm-call-id" not in request.headers
|
||||
assert "litellm-disable-message-redaction" not in request.headers
|
||||
assert int(request.headers["content-length"]) == len(await request.body())
|
||||
payload: Final = TypeAdapter(Mapping[str, object]).validate_json(await request.body())
|
||||
assert payload["metadata" if protocol == "chat" else "litellm_metadata"] == MappingProxyType(
|
||||
{"user_api_key_user_id": "fixture-user"}
|
||||
)
|
||||
with pytest.raises(BadRequestError, match="regular model group"):
|
||||
reject_recursive_compactor("auto-router")
|
||||
return MappingProxyType({"summary": "compacted"})
|
||||
|
||||
request: Final = _request(app, (
|
||||
(b"authorization", b"Bearer sk-compaction-fixture"), (b"cookie", b"session=fixture"),
|
||||
(b"content-length", b"99999"), (b"x-litellm-call-id", b"parent"),
|
||||
(b"litellm-disable-message-redaction", b"true"),
|
||||
))
|
||||
with inherit_message_logging_privacy(True):
|
||||
assert (await with_proxy_compaction_executor(_child(protocol), request))["summary"] == "compacted"
|
||||
assert get_request_stash() is stash and compaction_executor.get() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("chat", "messages"))
|
||||
async def test_real_proxy_denies_compactor_outside_key_models(
|
||||
monkeypatch: pytest.MonkeyPatch, protocol: Literal["chat", "messages"]
|
||||
) -> None:
|
||||
monkeypatch.setattr(proxy_server.app, "dependency_overrides", {})
|
||||
cache: Final = DualCache()
|
||||
token: Final = proxy_server.hash_token("sk-compaction-fixture")
|
||||
await cache.async_set_cache(key=token, value=UserAPIKeyAuth.model_validate(
|
||||
MappingProxyType({"token": token, "models": ("answer",)})
|
||||
))
|
||||
monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture")
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", object())
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", cache)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
paid_call: Final = AsyncMock(side_effect=AssertionError("Denied compactor reached inference"))
|
||||
monkeypatch.setattr(common_request_processing, "route_request", paid_call)
|
||||
request: Final = _request(proxy_server.app, ((b"authorization", b"Bearer sk-compaction-fixture"),))
|
||||
with pytest.raises(BadRequestError, match=r"child request failed \(HTTP 403\)"):
|
||||
await with_proxy_compaction_executor(_child(protocol), request)
|
||||
paid_call.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("forged_wire_flag", (False, True))
|
||||
@pytest.mark.parametrize("protocol", ("chat", "messages"))
|
||||
async def test_real_proxy_child_inherits_privacy_without_client_logging_controls(
|
||||
monkeypatch: pytest.MonkeyPatch, forged_wire_flag: bool, protocol: Literal["chat", "messages"]
|
||||
) -> None:
|
||||
monkeypatch.setattr(proxy_server.app, "dependency_overrides", {})
|
||||
dispatched: Final = asyncio.Event()
|
||||
|
||||
async def route(
|
||||
data: Mapping[str, object], llm_router: Router | None, user_model: str | None,
|
||||
route_type: str, user_api_key_dict: UserAPIKeyAuth | None,
|
||||
) -> Awaitable[ModelResponse]:
|
||||
dispatched.set()
|
||||
logging: Final = data["litellm_logging_obj"]
|
||||
assert isinstance(logging, Logging)
|
||||
assert logging.standard_callback_dynamic_params.get("turn_off_message_logging") is True
|
||||
assert should_redact_message_logging(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({
|
||||
"litellm_params": data, "standard_callback_dynamic_params": logging.standard_callback_dynamic_params,
|
||||
})))
|
||||
return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor"))
|
||||
|
||||
async def private_child() -> Mapping[str, object]:
|
||||
executor: Final = compaction_executor.get()
|
||||
assert executor is not None
|
||||
payload: Final = TypeAdapter(Mapping[str, object]).validate_json(
|
||||
b'{"model":"compactor","messages":[{"role":"user","content":"history"}],'
|
||||
b'"metadata":{"turn_off_message_logging":true}}'
|
||||
)
|
||||
return await executor(protocol, MappingProxyType({
|
||||
"litellm_metadata" if protocol == "messages" and key == "metadata" else key: value
|
||||
for key, value in payload.items()
|
||||
if forged_wire_flag or key != "metadata"
|
||||
}))
|
||||
|
||||
monkeypatch.setattr(proxy_server, "master_key", "sk-compaction-fixture")
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", TypeAdapter(dict[str, bool]).validate_python(
|
||||
MappingProxyType({})
|
||||
))
|
||||
monkeypatch.setattr(common_request_processing, "route_request", route)
|
||||
request: Final = _request(proxy_server.app, ((b"authorization", b"Bearer sk-compaction-fixture"),))
|
||||
with inherit_message_logging_privacy(True):
|
||||
if forged_wire_flag:
|
||||
with pytest.raises(BadRequestError, match=r"child request failed \(HTTP 401\)"):
|
||||
await with_proxy_compaction_executor(private_child(), request)
|
||||
assert not dispatched.is_set()
|
||||
return
|
||||
assert (await with_proxy_compaction_executor(private_child(), request))["id"] == "private-summary"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelling_parent_cancels_and_drains_child() -> None:
|
||||
app: Final = FastAPI()
|
||||
started: Final = asyncio.Event()
|
||||
stopped: Final = asyncio.Event()
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def endpoint() -> None:
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
stopped.set()
|
||||
|
||||
with inherit_message_logging_privacy(True):
|
||||
parent: Final = asyncio.create_task(with_proxy_compaction_executor(_child(), _request(app)))
|
||||
await asyncio.wait_for(started.wait(), timeout=5)
|
||||
parent.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await parent
|
||||
assert stopped.is_set() and compaction_executor.get() is None
|
||||
assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None
|
||||
|
|
@ -0,0 +1,357 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ContextCompactionConfig
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
CompactionState,
|
||||
_history,
|
||||
compact_to_fit,
|
||||
compaction_executor,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies
|
||||
from litellm.types.router import Deployment
|
||||
|
||||
pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_transport(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setenv("LITELLM_LICENSE", "")
|
||||
|
||||
|
||||
def make_router(
|
||||
*, mode: Literal["configured", "default", "empty", "disabled"] = "configured",
|
||||
compactor_window: int = 8192, target_window: int | None = 512, pin_output: bool = True,
|
||||
compactor_model: str = "anthropic/claude-sonnet-5",
|
||||
compactor_api_base: str | None = None,
|
||||
target_model: str = "anthropic/claude-haiku-4-5-20251001",
|
||||
target_output: int | None = 128,
|
||||
) -> litellm.Router:
|
||||
compaction: Final = {"configured": {"model": "large", "max_tokens": 512}, "empty": {}, "disabled": False}
|
||||
config: Final = {
|
||||
"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"},
|
||||
"keyword_tier_rules": [{"keywords": ["answer"], "tier": "SIMPLE"}],
|
||||
**({"context_compaction": compaction[mode]} if mode != "default" else {}),
|
||||
"enable_context_window_escalation": mode != "disabled",
|
||||
"max_tokens_from_tier_model": pin_output,
|
||||
}
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "auto", "litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": config}},
|
||||
{"model_name": "small", "litellm_params": {"model": target_model, "api_key": "test"}, "model_info": {"id": "pinned-small", "max_input_tokens": target_window, "max_output_tokens": target_output}},
|
||||
{"model_name": "large", "litellm_params": {"model": compactor_model, "api_key": "test", **({"api_base": compactor_api_base} if compactor_api_base else {})}, "model_info": {"id": "native-large", "max_input_tokens": compactor_window, "max_output_tokens": 4096}},
|
||||
],
|
||||
enable_pre_call_checks=True,
|
||||
num_retries=0,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
|
||||
|
||||
def history() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * 150},
|
||||
{"role": "assistant", "content": "Recorded"},
|
||||
{"role": "user", "content": "Answer with the project code"},
|
||||
]
|
||||
|
||||
|
||||
def provider_reply(payload: Mapping[str, object], summary: str = "Project code MAPLE-47") -> httpx.Response:
|
||||
compact: Final = "compaction" in payload
|
||||
return httpx.Response(200, json={
|
||||
"id": "msg_test", "type": "message", "role": "assistant", "model": payload["model"],
|
||||
"content": [{"type": "compaction", "content": summary, "signature": "native-signature"}] if compact else [{"type": "text", "text": "MAPLE-47"}],
|
||||
"stop_reason": "compaction" if compact else "end_turn",
|
||||
"usage": {"input_tokens": 0, "output_tokens": 0, "iterations": [{"type": "compaction", "input_tokens": 1200, "output_tokens": 20}]} if compact else {"input_tokens": 60, "output_tokens": 8},
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("messages_api", [False, True])
|
||||
@pytest.mark.parametrize("target_window", [512, 1640])
|
||||
@pytest.mark.parametrize(("mode", "gateway"), [("configured", False), ("default", False), ("empty", False), ("default", True)])
|
||||
async def test_router_compacts_on_native_model_and_answers_on_selected_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, messages_api: bool, target_window: int,
|
||||
mode: Literal["configured", "default", "empty"], gateway: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False)
|
||||
router: Final = make_router(
|
||||
target_window=target_window, mode=mode,
|
||||
compactor_model="openai/anthropic/claude-sonnet-5" if gateway else "anthropic/claude-sonnet-5",
|
||||
compactor_api_base="https://gateway.test/v1" if gateway else None,
|
||||
)
|
||||
messages: Final = history()
|
||||
original: Final = deepcopy(messages)
|
||||
expected: Final = messages if messages_api else [{**message, "content": [{"type": "text", "text": message["content"]}]} for message in messages]
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert "_context_compaction_state" not in payload
|
||||
if "compaction" in payload:
|
||||
assert payload["model"] == ("anthropic/claude-sonnet-5" if gateway else "claude-sonnet-5")
|
||||
assert payload["compaction"] == {"type": "summarize"}
|
||||
assert "compact-2026-09-04" in request.headers.get("anthropic-beta", "").split(",")
|
||||
assert payload["messages"] == (messages if gateway else expected)[:-1]
|
||||
assert payload["max_tokens"] == (512 if mode == "configured" else 4096)
|
||||
if gateway:
|
||||
return httpx.Response(200, json={
|
||||
"id": "summary", "model": payload["model"], "object": "chat.completion", "created": 0,
|
||||
"choices": [{"index": 0, "finish_reason": "stop", "message": {
|
||||
"role": "assistant", "content": "", "provider_specific_fields": {
|
||||
"compaction_blocks": [{"type": "compaction", "content": "Project code MAPLE-47", "signature": "native-signature"}],
|
||||
},
|
||||
}}],
|
||||
"usage": {"prompt_tokens": 1200, "completion_tokens": 20, "total_tokens": 1220},
|
||||
})
|
||||
else:
|
||||
assert payload["model"] == "claude-haiku-4-5-20251001"
|
||||
assert payload["messages"][-1] == expected[-1]
|
||||
assert "MAPLE-47" in str(payload["messages"][0])
|
||||
assert "signature" not in str(payload["messages"])
|
||||
return provider_reply(payload)
|
||||
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
route: Final = transport.post("https://api.anthropic.com/v1/messages").mock(side_effect=respond)
|
||||
gateway_route: Final = transport.post("https://gateway.test/v1/chat/completions").mock(side_effect=respond)
|
||||
call: Final = router.aanthropic_messages if messages_api else router.acompletion
|
||||
response: Final = await call(model="auto", messages=messages, max_tokens=64)
|
||||
assert route.call_count == (1 if gateway else 2)
|
||||
assert gateway_route.call_count == int(gateway)
|
||||
assert response is not None
|
||||
assert messages == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("messages_api", [False, True])
|
||||
@pytest.mark.parametrize("capability", [True, False, None])
|
||||
async def test_catalog_compactor_support_is_independent_of_deepseek_answering_model(
|
||||
monkeypatch: pytest.MonkeyPatch, messages_api: bool, capability: bool | None,
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "native-summary-fixture", {
|
||||
"litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 8192,
|
||||
"max_output_tokens": 4096, "supports_anthropic_compaction": capability,
|
||||
})
|
||||
router: Final = make_router(
|
||||
mode="default", compactor_model="anthropic/native-summary-fixture",
|
||||
target_model="deepseek/deepseek-v4-flash",
|
||||
)
|
||||
|
||||
def summarize(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert payload["model"] == "native-summary-fixture"
|
||||
assert payload["compaction"] == {"type": "summarize"}
|
||||
return provider_reply(payload)
|
||||
|
||||
def answer(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert payload["model"] == "deepseek-v4-flash"
|
||||
assert "Project code MAPLE-47" in str(payload["messages"][0])
|
||||
assert payload["messages"][-1] == history()[-1]
|
||||
assert "compaction" not in payload and "signature" not in str(payload["messages"])
|
||||
if messages_api:
|
||||
return provider_reply(payload)
|
||||
return httpx.Response(200, json={
|
||||
"id": "answer", "model": payload["model"], "object": "chat.completion", "created": 0,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "MAPLE-47"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 60, "completion_tokens": 8, "total_tokens": 68},
|
||||
})
|
||||
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
native: Final = transport.post("https://api.anthropic.com/v1/messages").mock(side_effect=summarize)
|
||||
target: Final = transport.post("https://api.deepseek.com/anthropic/v1/messages" if messages_api else "https://api.deepseek.com/beta/chat/completions").mock(side_effect=answer)
|
||||
call: Final = router.aanthropic_messages if messages_api else router.acompletion
|
||||
if capability:
|
||||
await call(model="auto", messages=history(), max_tokens=64)
|
||||
else:
|
||||
with pytest.raises(litellm.BadRequestError, match="No configured tier model"):
|
||||
await call(model="auto", messages=history(), max_tokens=64)
|
||||
assert native.call_count == target.call_count == int(capability is True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["no_history", "compactor_too_small", "summary_too_large", "retained_too_large", "not_native", "empty_summary", "invalid_window"])
|
||||
async def test_overflow_fails_without_sending_an_oversized_answer(failure: str) -> None:
|
||||
router: Final = make_router(compactor_window=800 if failure == "compactor_too_small" else 8192, target_window=0 if failure == "invalid_window" else 512)
|
||||
messages: Final = history()[:1] if failure == "no_history" else [*history()[:-1], {"role": "user", "content": "Answer " * 1500}] if failure == "retained_too_large" else history()
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert "compaction" in payload
|
||||
return provider_reply(
|
||||
{"model": payload["model"]} if failure == "not_native" else payload,
|
||||
summary="" if failure == "empty_summary" else "oversized " * 1500,
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
route: Final = transport.post("https://api.anthropic.com/v1/messages").mock(side_effect=respond)
|
||||
with pytest.raises((litellm.BadRequestError, litellm.ContextWindowExceededError)):
|
||||
await router.acompletion(model="auto", messages=messages, max_tokens=64)
|
||||
assert route.call_count == (1 if failure in ("summary_too_large", "not_native", "empty_summary") else 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_opt_out_disables_compaction_without_changing_existing_routing() -> None:
|
||||
router: Final = make_router(mode="disabled")
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
route: Final = transport.post("https://api.anthropic.com/v1/messages").mock(return_value=provider_reply({"model": "claude-sonnet-5"}))
|
||||
await router.acompletion(model="auto", messages=history(), max_tokens=64)
|
||||
assert route.call_count == 1
|
||||
assert "compaction" not in json.loads(route.calls[0].request.content)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled", [False, None])
|
||||
def test_compaction_opt_out_survives_management_config_serialization(disabled: Literal[False] | None) -> None:
|
||||
config: Final = ComplexityRouterConfig(context_compaction=disabled)
|
||||
restored: Final = ComplexityRouterConfig.model_validate(config.model_dump(exclude_none=True))
|
||||
assert restored.context_compaction is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("model", "window", "mixed"), (
|
||||
("anthropic/claude-haiku-4-5-20251001", 8192, False),
|
||||
("anthropic/claude-sonnet-5", 800, False),
|
||||
("anthropic/claude-sonnet-5", 8192, True),
|
||||
))
|
||||
async def test_default_compaction_rejects_ineligible_tier_compactors_before_spending(model: str, window: int, mixed: bool) -> None:
|
||||
router: Final = make_router(mode="default", compactor_model=model, compactor_window=window)
|
||||
if mixed:
|
||||
router.add_deployment(Deployment.model_validate({
|
||||
"model_name": "large", "litellm_params": {"model": model, "api_key": "test"},
|
||||
"model_info": {"id": "undersized-member", "max_input_tokens": 800, "max_output_tokens": 512},
|
||||
}))
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
route: Final = transport.post("https://api.anthropic.com/v1/messages")
|
||||
with pytest.raises(litellm.BadRequestError, match="No configured tier model"):
|
||||
await router.acompletion(model="auto", messages=history(), max_tokens=64)
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("oversized", [False, True])
|
||||
@pytest.mark.parametrize(("messages_api", "output"), (
|
||||
(False, None), (False, 460), (False, 512), (True, 460), (True, 512),
|
||||
))
|
||||
async def test_unusable_reserved_budget_does_not_bypass_known_input_window(
|
||||
oversized: bool, messages_api: bool, output: int | None,
|
||||
) -> None:
|
||||
router: Final = make_router(
|
||||
mode="default", pin_output=False, target_model="anthropic/unknown-output-fixture", target_output=None,
|
||||
)
|
||||
messages: Final = history() if oversized else [{"role": "user", "content": "Answer hello"}]
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
route: Final = transport.post("https://api.anthropic.com/v1/messages").mock(return_value=provider_reply({"model": "unknown-output-fixture"}))
|
||||
call: Final = router.aanthropic_messages if messages_api else router.acompletion
|
||||
if oversized:
|
||||
with pytest.raises(litellm.BadRequestError, match="known input window"):
|
||||
await call(model="auto", messages=messages, max_tokens=output)
|
||||
else:
|
||||
await call(model="auto", messages=messages, max_tokens=output)
|
||||
assert route.call_count == int(not oversized)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("target_window", [None, 512])
|
||||
@pytest.mark.parametrize(("messages_api", "output"), (
|
||||
(False, {"max_tokens": 64}),
|
||||
(False, {"max_tokens": 64, "max_completion_tokens": None}),
|
||||
(False, {"max_tokens": None}),
|
||||
(False, {"max_tokens": 460}),
|
||||
(False, {"max_tokens": 512}),
|
||||
(True, {"max_tokens": 64}),
|
||||
(True, {"max_tokens": 64, "max_completion_tokens": None}),
|
||||
(True, {"max_tokens": 460}),
|
||||
(True, {"max_tokens": 512}),
|
||||
))
|
||||
async def test_fitting_or_unknown_budget_requests_do_not_call_compactor(messages_api: bool, output: Mapping[str, int | None], target_window: int | None) -> None:
|
||||
router: Final = make_router(mode="default", pin_output=False, target_window=target_window, target_model="anthropic/unknown-window-fixture", compactor_model="anthropic/claude-haiku-4-5-20251001")
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
route: Final = transport.post("https://api.anthropic.com/v1/messages").mock(return_value=provider_reply({"model": "claude-haiku-4-5-20251001"}))
|
||||
call: Final = router.aanthropic_messages if messages_api else router.acompletion
|
||||
await call(model="auto", messages=history() if target_window is None else [{"role": "user", "content": "Answer hello"}], **output)
|
||||
assert route.call_count == 1
|
||||
assert "compaction" not in json.loads(route.calls[0].request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("privacy", "settings", "global_privacy"), (
|
||||
(True, {"turn_off_message_logging": True}, False),
|
||||
(True, {"metadata": {"turn_off_message_logging": True}}, False),
|
||||
(True, {}, True),
|
||||
(True, {"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}, False),
|
||||
(False, {"turn_off_message_logging": False}, False),
|
||||
(False, {"metadata": {"headers": {"litellm-disable-message-redaction": "true"}}}, True),
|
||||
))
|
||||
async def test_summary_reused_on_retry_and_includes_system_and_tools_in_budget(
|
||||
monkeypatch: pytest.MonkeyPatch, privacy: bool, settings: Mapping[str, object], global_privacy: bool,
|
||||
) -> None:
|
||||
router: Final = make_router()
|
||||
deployment: Final = router.get_deployment(model_id="pinned-small").model_dump()
|
||||
state: Final = CompactionState(config=ContextCompactionConfig(model="large", max_tokens=512))
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", global_privacy)
|
||||
payload: Final = {
|
||||
"model": "small", "messages": history(), "max_tokens": 64, "_context_compaction_state": state, **settings,
|
||||
}
|
||||
calls: Final = asyncio.Queue[Mapping[str, object]]()
|
||||
|
||||
async def execute(protocol: str, request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
assert "turn_off_message_logging" not in request and "turn_off_message_logging" not in request["metadata"]
|
||||
assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging", False) is privacy
|
||||
calls.put_nowait(request)
|
||||
return {"choices": [{"message": {"provider_specific_fields": {"compaction_blocks": [{"type": "compaction", "content": "MAPLE-47", "signature": "signed"}]}}}]}
|
||||
|
||||
token: Final = compaction_executor.set(execute)
|
||||
try:
|
||||
first: Final = await compact_to_fit(router, deployment, payload, "chat")
|
||||
second: Final = await compact_to_fit(router, deployment, payload, "chat")
|
||||
assert first == second and calls.qsize() == 1
|
||||
with pytest.raises(litellm.BadRequestError, match="leave no room"):
|
||||
await compact_to_fit(router, deployment, {**payload, "max_tokens": 450}, "chat")
|
||||
with pytest.raises(litellm.BadRequestError, match="leave no room"):
|
||||
await compact_to_fit(router, deployment, {**payload, "system": "instructions " * 600}, "chat")
|
||||
finally:
|
||||
compaction_executor.reset(token)
|
||||
assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compaction_timeout_is_replayed_without_cancelling_retry_or_spending_again() -> None:
|
||||
router: Final = make_router()
|
||||
deployment: Final = router.get_deployment(model_id="pinned-small").model_dump()
|
||||
state: Final = CompactionState(config=ContextCompactionConfig(model="large", timeout_seconds=0.01))
|
||||
payload: Final = {"model": "small", "messages": history(), "max_tokens": 64, "_context_compaction_state": state}
|
||||
calls: Final = asyncio.Queue[None]()
|
||||
stopped: Final = asyncio.Event()
|
||||
|
||||
async def execute(protocol: str, request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
calls.put_nowait(None)
|
||||
try:
|
||||
return await asyncio.Future[Mapping[str, object]]()
|
||||
finally:
|
||||
stopped.set()
|
||||
|
||||
token: Final = compaction_executor.set(execute)
|
||||
try:
|
||||
for _attempt in range(2):
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await compact_to_fit(router, deployment, payload, "chat")
|
||||
assert calls.qsize() == 1 and stopped.is_set()
|
||||
finally:
|
||||
compaction_executor.reset(token)
|
||||
|
||||
|
||||
def test_native_tool_result_tail_is_retained_and_compactor_is_an_authorized_dependency() -> None:
|
||||
messages: Final = [*history(), {"role": "assistant", "content": [{"type": "tool_use", "id": "call", "name": "lookup", "input": {}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call", "content": "value"}]}]
|
||||
_, prefix, tail = _history(messages, "small")
|
||||
assert list(prefix) == messages[:2]
|
||||
assert list(tail) == messages[2:]
|
||||
dependencies: Final = strategy_router_dependencies({"model": "auto_router/complexity_router", "complexity_router_config": {"context_compaction": {"model": "large"}}})
|
||||
assert [(dependency.model_name, dependency.role) for dependency in dependencies] == [("large", "compactor")]
|
||||
|
|
@ -14894,7 +14894,9 @@ class TestHealthFallbackDispatch:
|
|||
) -> None:
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
||||
router: Final = self._router(config={"tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}})
|
||||
router: Final = self._router(
|
||||
config={"context_compaction": False, "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}}
|
||||
)
|
||||
router.add_deployment(
|
||||
Deployment(
|
||||
model_name="large",
|
||||
|
|
@ -14971,7 +14973,9 @@ class TestHealthFallbackDispatch:
|
|||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("default_fits", [True, False])
|
||||
async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None:
|
||||
router: Final = self._router(config={"modality_routing": True, "tiers": {"SIMPLE": "primary"}})
|
||||
router: Final = self._router(
|
||||
config={"context_compaction": False, "modality_routing": True, "tiers": {"SIMPLE": "primary"}}
|
||||
)
|
||||
for deployment in router.model_list:
|
||||
deployment["model_info"]["supports_vision"] = deployment["model_name"] == "fallback"
|
||||
deployment["model_info"]["max_input_tokens"] = 10000 if default_fits else 10
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ class TestAnthropicBetaHeadersFiltering:
|
|||
filtered = filter_and_transform_beta_headers(
|
||||
beta_headers=all_headers, provider=provider
|
||||
)
|
||||
assert ("compact-2026-09-04" in filtered) is (provider == "anthropic")
|
||||
|
||||
for header in unsupported_headers:
|
||||
assert (
|
||||
|
|
|
|||
|
|
@ -855,6 +855,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"source": {"type": "string"},
|
||||
"comment": {"type": "string"},
|
||||
"supports_assistant_prefill": {"type": "boolean"},
|
||||
"supports_anthropic_compaction": {"type": "boolean"},
|
||||
"supports_audio_input": {"type": "boolean"},
|
||||
"supports_audio_output": {"type": "boolean"},
|
||||
"gemini_native_audio": {"type": "boolean"},
|
||||
|
|
|
|||
25
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
25
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -27316,6 +27316,26 @@ export interface components {
|
|||
*/
|
||||
pattern_type: "prebuilt" | "regex";
|
||||
};
|
||||
/** ContextCompactionConfig */
|
||||
ContextCompactionConfig: {
|
||||
/**
|
||||
* Max Tokens
|
||||
* @default 4096
|
||||
*/
|
||||
max_tokens: number;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
/**
|
||||
* Timeout Seconds
|
||||
* @default 120
|
||||
*/
|
||||
timeout_seconds: number;
|
||||
/**
|
||||
* Trigger Ratio
|
||||
* @default 0.9
|
||||
*/
|
||||
trigger_ratio: number;
|
||||
};
|
||||
/**
|
||||
* CoordinationRedisNode
|
||||
* @description A single startup node of a cluster-mode Redis used for proxy coordination.
|
||||
|
|
@ -36462,6 +36482,11 @@ export interface components {
|
|||
* @description Keywords indicating code-related content
|
||||
*/
|
||||
code_keywords?: string[] | null;
|
||||
/**
|
||||
* Context Compaction
|
||||
* @description Native Anthropic compact-to-fit is enabled by default for Chat and Messages. Selects a compatible configured tier model unless model is specified. Set false to disable (null is normalized to false). Takes precedence over context-window escalation on supported requests.
|
||||
*/
|
||||
context_compaction?: components["schemas"]["ContextCompactionConfig"] | false | null;
|
||||
/**
|
||||
* Context Window Escalation Buffer
|
||||
* @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue