mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(router): native compact-to-fit across conversation APIs
This commit is contained in:
parent
daecea3eb8
commit
d1a74f0214
33 changed files with 1545 additions and 47 deletions
|
|
@ -7,6 +7,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,24 @@
|
|||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
|
||||
|
||||
_CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
|
||||
_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
|
||||
"inherited_message_logging_disabled", default=False
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
|
||||
token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_inherited_message_logging_disabled.reset(token)
|
||||
|
||||
|
||||
def iter_client_callback_metadata_dicts(
|
||||
|
|
@ -143,7 +157,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple
|
|||
|
||||
|
||||
def initialize_standard_callback_dynamic_params(
|
||||
kwargs: dict | None = None,
|
||||
kwargs: dict[str, object] | None = None,
|
||||
) -> StandardCallbackDynamicParams:
|
||||
"""
|
||||
Initialize the standard callback dynamic params from the kwargs
|
||||
|
|
@ -179,4 +193,10 @@ def initialize_standard_callback_dynamic_params(
|
|||
if param in _trusted_overlay_callback_params:
|
||||
standard_callback_dynamic_params[param] = trusted_value
|
||||
|
||||
if _inherited_message_logging_disabled.get():
|
||||
private_params: Final[StandardCallbackDynamicParams] = {
|
||||
**standard_callback_dynamic_params,
|
||||
"turn_off_message_logging": True,
|
||||
}
|
||||
return private_params
|
||||
return standard_callback_dynamic_params
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ from ..common_utils import (
|
|||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
)
|
||||
|
||||
|
|
@ -1771,7 +1772,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
return tools
|
||||
|
||||
def _ensure_beta_header(self, headers: dict, beta_value: str) -> None:
|
||||
def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None:
|
||||
"""
|
||||
Ensure a beta header value is present in the anthropic-beta header.
|
||||
Merges with existing values instead of overriding them.
|
||||
|
|
@ -1780,13 +1781,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers: Dictionary of headers to update
|
||||
beta_value: The beta header value to add
|
||||
"""
|
||||
existing_beta: Final = headers.get("anthropic-beta")
|
||||
if existing_beta is None:
|
||||
headers["anthropic-beta"] = beta_value
|
||||
return
|
||||
existing_values: Final = [beta.strip() for beta in existing_beta.split(",")]
|
||||
if beta_value not in existing_values:
|
||||
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
|
||||
existing_values: Final = tuple(
|
||||
beta.strip()
|
||||
for key, value in headers.items()
|
||||
if key.lower() == "anthropic-beta"
|
||||
for beta in value.split(",")
|
||||
if beta.strip()
|
||||
)
|
||||
for key in tuple(headers):
|
||||
if key.lower() == "anthropic-beta":
|
||||
headers.pop(key)
|
||||
headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value)))
|
||||
|
||||
def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None:
|
||||
"""
|
||||
|
|
@ -1824,7 +1829,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
)
|
||||
|
||||
def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict:
|
||||
def update_headers_with_optional_anthropic_beta(
|
||||
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
|
||||
) -> dict:
|
||||
"""Update headers with optional anthropic beta."""
|
||||
|
||||
# Skip adding beta headers for Vertex requests
|
||||
|
|
@ -1833,6 +1840,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if is_vertex_request:
|
||||
return headers
|
||||
|
||||
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
|
||||
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
_tools: Final = optional_params.get("tools", [])
|
||||
for tool in _tools:
|
||||
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
|
||||
|
|
@ -1929,8 +1939,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
custom_llm_provider=self._resolved_provider,
|
||||
)
|
||||
|
||||
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
|
||||
|
||||
# === Tool-name sanitization (single chokepoint) ===
|
||||
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
|
||||
# sanitize *here* -- not in map_openai_params -- because:
|
||||
|
|
@ -1977,6 +1985,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
message=f"{e}\nReceived Messages={messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
headers=headers, optional_params=optional_params, messages=anthropic_messages
|
||||
)
|
||||
|
||||
## Auto-strip advisor blocks from history if advisor tool is absent.
|
||||
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
|
||||
_all_tools: Final = optional_params.get("tools") or []
|
||||
|
|
|
|||
|
|
@ -77,6 +77,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
|||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
||||
|
||||
def requires_native_compaction_beta(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and bool(block.get("signature"))
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, (list, tuple))
|
||||
for block in content
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
|
|||
92
litellm/llms/anthropic/compaction.py
Normal file
92
litellm/llms/anthropic/compaction.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
Protocol: TypeAlias = Literal["chat", "messages"]
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_MESSAGES: Final = TypeAdapter(list[AllMessageValues])
|
||||
_HEADERS: Final = TypeAdapter(dict[str, str])
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice")
|
||||
|
||||
|
||||
def supports_native_compaction(params: Mapping[str, object]) -> bool:
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
if params.get("custom_llm_provider") not in (None, "anthropic", "openai"):
|
||||
return False
|
||||
model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/")
|
||||
try:
|
||||
return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def compatible_defaults(payload: Mapping[str, object]) -> bool:
|
||||
return all(payload.get(key) is None for key in _CONFLICTS)
|
||||
|
||||
|
||||
def request_kwargs() -> Mapping[str, object]:
|
||||
operation: Final[AnthropicCompaction] = {"type": "summarize"}
|
||||
return MappingProxyType(
|
||||
{
|
||||
"compaction": operation,
|
||||
"extra_headers": _HEADERS.validate_python(
|
||||
MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _native_blocks(protocol: Protocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
if protocol == "messages":
|
||||
return (
|
||||
_OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else ()
|
||||
)
|
||||
choices: Final = _OBJECTS.validate_python(response.get("choices", ()))
|
||||
choice: Final = choices[0] if len(choices) == 1 else _EMPTY
|
||||
message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY))
|
||||
fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY)
|
||||
return _OBJECTS.validate_python(fields.get("compaction_blocks", ()))
|
||||
|
||||
|
||||
def extract_summary(protocol: Protocol, response: Mapping[str, object]) -> str | None:
|
||||
blocks: Final = _native_blocks(protocol, response)
|
||||
block: Final = blocks[0] if len(blocks) == 1 else _EMPTY
|
||||
content: Final = block.get("content")
|
||||
return (
|
||||
content
|
||||
if block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and block.get("signature")
|
||||
and isinstance(content, str)
|
||||
and content.strip()
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
async def dispatch(router: Router, protocol: Protocol, payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if protocol == "messages":
|
||||
return _MAPPING.validate_python(
|
||||
await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model=str(payload["model"]),
|
||||
messages=_MESSAGES.validate_python(payload["messages"]),
|
||||
stream=False,
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
|
||||
),
|
||||
)
|
||||
return _MAPPING.validate_python(response.model_dump())
|
||||
|
|
@ -4,6 +4,8 @@ import json
|
|||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
|
|
@ -16,6 +18,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
|
|||
TOOL_NAME_HASH_LENGTH: Final = 8
|
||||
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
|
||||
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
|
||||
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _optional_attr(source: object, name: str) -> object:
|
||||
|
|
@ -36,6 +39,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
|
|||
return signature if isinstance(signature, str) else None
|
||||
|
||||
|
||||
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = _as_string_mapping(provider_specific_fields)
|
||||
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
return (
|
||||
tuple(
|
||||
block
|
||||
for raw_block in raw_blocks
|
||||
if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction"
|
||||
)
|
||||
if isinstance(raw_blocks, (list, tuple))
|
||||
else ()
|
||||
)
|
||||
|
||||
|
||||
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
|
||||
{"name", "type", "input_schema", "description", "cache_control", "strict"}
|
||||
)
|
||||
|
|
@ -1330,7 +1347,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_name_mapping: dict[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
new_content: Final[list[dict[str, Any]]] = []
|
||||
for choice in choices:
|
||||
for choice, compaction_blocks in (
|
||||
(choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))
|
||||
for choice in choices
|
||||
):
|
||||
new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks)
|
||||
# Handle thinking blocks first
|
||||
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
|
||||
for thinking_block in choice.message.thinking_blocks:
|
||||
|
|
@ -1365,7 +1386,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
# Handle text content
|
||||
if choice.message.content is not None:
|
||||
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
|
||||
new_content.append(
|
||||
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
|
||||
)
|
||||
|
|
@ -1545,21 +1566,31 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
openai_finish_reason=openai_finish_reason
|
||||
)
|
||||
anthropic_finish_reason: Final = (
|
||||
"refusal"
|
||||
"compaction"
|
||||
if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction"
|
||||
else "refusal"
|
||||
if refusal_text is not None and translated_finish_reason != "max_tokens"
|
||||
else translated_finish_reason
|
||||
)
|
||||
# extract usage
|
||||
usage: Final[Usage] = getattr(response, "usage")
|
||||
anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
|
||||
|
||||
if polyfill_result is not None and polyfill_result.iterations_usage is not None:
|
||||
message_iteration: Final[UsageIteration] = {
|
||||
"type": "message",
|
||||
"input_tokens": anthropic_usage["input_tokens"],
|
||||
"output_tokens": usage.completion_tokens or 0,
|
||||
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
|
||||
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
|
||||
anthropic_usage: Final[AnthropicUsage] = (
|
||||
{
|
||||
**message_usage,
|
||||
"iterations": [
|
||||
*polyfill_iterations,
|
||||
UsageIteration(
|
||||
type="message",
|
||||
input_tokens=message_usage.get("input_tokens", 0),
|
||||
output_tokens=usage.completion_tokens or 0,
|
||||
),
|
||||
],
|
||||
}
|
||||
anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration]
|
||||
if polyfill_iterations is not None
|
||||
else message_usage
|
||||
)
|
||||
|
||||
translated_obj: Final = AnthropicMessagesResponse(
|
||||
id=response.id,
|
||||
|
|
|
|||
|
|
@ -580,7 +580,9 @@ def anthropic_messages_handler(
|
|||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
|
||||
if kwargs.get("compaction") is None and _should_route_to_responses_api(
|
||||
custom_llm_provider, original_model, model
|
||||
):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from ...common_utils import (
|
|||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
|
@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
@ -633,6 +635,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
beta_values.update(existing_beta)
|
||||
|
||||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
|
|
|
|||
|
|
@ -14487,6 +14487,7 @@
|
|||
"source": "https://docs.anthropic.com/en/docs/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 4e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -14526,6 +14527,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 6e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
|
|
@ -14741,6 +14743,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14779,6 +14782,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6-20260205": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14816,6 +14820,7 @@
|
|||
"prompt_cache_min_tokens": 4096
|
||||
},
|
||||
"claude-opus-4-7": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14855,6 +14860,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-7-20260416": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14893,6 +14899,7 @@
|
|||
"prompt_cache_min_tokens": 2048
|
||||
},
|
||||
"claude-fable-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -14932,6 +14939,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-fable-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -14972,6 +14980,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -15014,6 +15023,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-8": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -60334,6 +60344,7 @@
|
|||
"supports_audio_output": true
|
||||
},
|
||||
"claude-mythos-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -60373,6 +60384,7 @@
|
|||
}
|
||||
},
|
||||
"claude-mythos-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -60413,6 +60425,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-mythos-preview": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
|
|||
)
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import route_request
|
||||
from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails
|
||||
from litellm.router import Router
|
||||
|
|
@ -2533,7 +2534,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
user_model=user_model,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
llm_call_task: Final = asyncio.create_task(llm_call)
|
||||
llm_call_task: Final = asyncio.create_task(with_proxy_compaction_executor(llm_call, request))
|
||||
tasks.append(llm_call_task)
|
||||
|
||||
llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call
|
||||
|
|
|
|||
91
litellm/proxy/native_compaction.py
Normal file
91
litellm/proxy/native_compaction.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import asyncio
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from contextvars import Context
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
import httpx
|
||||
from fastapi import Request
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
compaction_executor,
|
||||
native_compaction_call,
|
||||
)
|
||||
|
||||
_ResultT: Final = TypeVar("_ResultT")
|
||||
_ASGI_APP: Final = TypeAdapter[ASGIApp](ASGIApp)
|
||||
_ROOT_PATH: Final = TypeAdapter(str)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
_REMOVED_HEADERS: Final = frozenset(
|
||||
(
|
||||
b"content-length",
|
||||
b"x-litellm-call-id",
|
||||
b"x-litellm-num-retries",
|
||||
b"x-litellm-timeout",
|
||||
b"x-litellm-stream-timeout",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def with_proxy_compaction_executor(call: Awaitable[_ResultT], request: Request) -> _ResultT:
|
||||
async def execute(protocol: Literal["chat", "messages"], payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
logging_disabled: Final = initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is True
|
||||
|
||||
async def dispatch() -> Mapping[str, object]:
|
||||
scope: Final = _JSON_OBJECT.validate_python(request.scope)
|
||||
root_path: Final = _ROOT_PATH.validate_python(scope.get("root_path", ""))
|
||||
path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages"
|
||||
url: Final = str(request.url.replace(path=root_path.rstrip("/") + path, query="", fragment=""))
|
||||
headers: Final = tuple(
|
||||
(name, value)
|
||||
for name, value in request.headers.raw
|
||||
if name.lower() not in _REMOVED_HEADERS
|
||||
and not (logging_disabled and name.decode("latin-1").lower() in UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS)
|
||||
)
|
||||
transport: Final = httpx.ASGITransport(
|
||||
app=_ASGI_APP.validate_python(scope["app"]),
|
||||
root_path=root_path,
|
||||
client=request.client, # pyright: ignore[reportArgumentType] # ASGI permits a missing client address
|
||||
raise_app_exceptions=False,
|
||||
)
|
||||
with native_compaction_call(), inherit_message_logging_privacy(logging_disabled):
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
follow_redirects=False,
|
||||
timeout=httpx.Timeout(None),
|
||||
) as client:
|
||||
async with client.stream(
|
||||
"POST", url, headers=headers, json=_JSON_OBJECT.validate_python(payload)
|
||||
) as response:
|
||||
if not response.is_success:
|
||||
raise BadRequestError(
|
||||
message=f"Native compaction child request failed (HTTP {response.status_code})",
|
||||
model="context_compaction",
|
||||
llm_provider="",
|
||||
)
|
||||
body: Final = await response.aread()
|
||||
try:
|
||||
return MappingProxyType(_JSON_OBJECT.validate_json(body))
|
||||
except ValidationError:
|
||||
raise BadRequestError(
|
||||
message="Native compaction child returned an invalid JSON object",
|
||||
model="context_compaction",
|
||||
llm_provider="",
|
||||
) from None
|
||||
|
||||
task: Final = Context().run(asyncio.create_task, dispatch())
|
||||
return await task
|
||||
|
||||
token: Final = compaction_executor.set(execute)
|
||||
try:
|
||||
return await call
|
||||
finally:
|
||||
compaction_executor.reset(token)
|
||||
|
|
@ -118,6 +118,15 @@ from litellm.llms.openai_like.model_info import (
|
|||
get_openai_compatible_model_info,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
arm_compaction,
|
||||
compact_to_fit,
|
||||
compaction_pending,
|
||||
initialize_compaction_state,
|
||||
is_native_compaction_call,
|
||||
reject_recursive_compactor,
|
||||
surface_for_call,
|
||||
)
|
||||
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
|
||||
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
|
||||
|
|
@ -3636,6 +3645,7 @@ class Router:
|
|||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat")
|
||||
async with contextlib.AsyncExitStack() as deployment_slot:
|
||||
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
|
||||
deployment_slot.enter_context(max_parallel_requests_limit)
|
||||
|
|
@ -3644,7 +3654,7 @@ class Router:
|
|||
logging_obj=logging_obj,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
response = await litellm.acompletion(**input_kwargs)
|
||||
response = await litellm.acompletion(**compacted_input)
|
||||
|
||||
## CHECK CONTENT FILTER ERROR ##
|
||||
if isinstance(response, ModelResponse):
|
||||
|
|
@ -5242,8 +5252,14 @@ class Router:
|
|||
if custom_llm_provider is not None:
|
||||
response_kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
compacted_input: Final = await compact_to_fit(
|
||||
self,
|
||||
deployment,
|
||||
response_kwargs,
|
||||
surface_for_call(getattr(original_generic_function, "__name__", "")),
|
||||
)
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await original_generic_function(**response_kwargs)
|
||||
response = await original_generic_function(**compacted_input)
|
||||
|
||||
if self._should_raise_anthropic_refusal_error(
|
||||
model=model,
|
||||
|
|
@ -7401,6 +7417,13 @@ class Router:
|
|||
If it fails after num_retries, fall back to another model group
|
||||
"""
|
||||
model_group: Final[str | None] = kwargs.get("model")
|
||||
if (
|
||||
surface_for_call(
|
||||
getattr(kwargs.get("original_generic_function") or kwargs.get("original_function"), "__name__", "")
|
||||
)
|
||||
is not None
|
||||
):
|
||||
kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs)
|
||||
clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary
|
||||
if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets):
|
||||
_fallback_metadata_key: Final = _get_router_metadata_variable_name(
|
||||
|
|
@ -12109,8 +12132,8 @@ class Router:
|
|||
|
||||
def _count_pre_call_check_tokens(
|
||||
self,
|
||||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
messages: Sequence[Mapping[str, object]] | None,
|
||||
input: str | list[object] | None,
|
||||
request_kwargs: Mapping[str, object] | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
|
|
@ -12257,7 +12280,9 @@ class Router:
|
|||
_rate_limit_error = False
|
||||
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs)
|
||||
|
||||
has_countable_input: Final = messages is not None or input is not None
|
||||
has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending(
|
||||
request_kwargs
|
||||
)
|
||||
|
||||
## get model group RPM ##
|
||||
dt: Final = get_utc_datetime()
|
||||
|
|
@ -13456,6 +13481,8 @@ class Router:
|
|||
registered_model_name: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> str:
|
||||
if is_native_compaction_call():
|
||||
return registered_model_name
|
||||
if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)):
|
||||
return registered_model_name
|
||||
cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs)
|
||||
|
|
@ -13534,6 +13561,7 @@ class Router:
|
|||
model=registered_model_name, request_kwargs=request_kwargs
|
||||
)
|
||||
if selected_strategy is None:
|
||||
arm_compaction(request_kwargs, None)
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
self._stamp_or_clear_metadata_key(
|
||||
request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
|
||||
|
|
@ -13544,6 +13572,24 @@ class Router:
|
|||
return None
|
||||
|
||||
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
|
||||
reject_recursive_compactor(registered_model_name)
|
||||
arm_compaction(
|
||||
request_kwargs,
|
||||
selected_strategy.strategy.config.context_compaction
|
||||
if isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
else None,
|
||||
tuple(
|
||||
dict.fromkeys(
|
||||
member
|
||||
for pool in selected_strategy.strategy.config.tiers.values()
|
||||
for member in ((pool,) if isinstance(pool, str) else pool)
|
||||
)
|
||||
)
|
||||
if isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
else (),
|
||||
)
|
||||
|
||||
await authorize_member_auto_router_inference(
|
||||
deployment=self._selected_strategy_marker_deployment(
|
||||
|
|
|
|||
|
|
@ -400,6 +400,31 @@ affinity write happens upstream of the gate and stores the session's own model,
|
|||
turn replays the original pin and the override is never pinned in its place. It does nothing
|
||||
unless `modality_routing` is also on.
|
||||
|
||||
### Native compact-to-fit
|
||||
|
||||
The complexity router keeps the selected answering deployment when full conversation history approaches its input limit. This is enabled by default for async `/v1/chat/completions`, `/v1/responses` and `/v1/messages`. It takes precedence over context-window escalation for those requests
|
||||
|
||||
The router selects a native-capable compactor from its configured tier groups, checks that the older history fits, and requests Anthropic's on-demand native compaction. It inserts the returned summary as ordinary assistant text, preserving instructions, tools and the latest user turn. The answering model can be Haiku, DeepSeek, Kimi, GPT or another configured model; it does not need to support compaction itself. A larger context window alone does not make a model a native compactor
|
||||
|
||||
Native capability comes from `supports_anthropic_compaction` in model metadata. Selection skips groups containing an unsupported, undersized or incompatibly configured deployment. No unrelated gateway model is selected automatically. To name a separate compactor or change the defaults:
|
||||
|
||||
```yaml
|
||||
complexity_router_config:
|
||||
context_compaction:
|
||||
model: native-compactor
|
||||
trigger_ratio: 0.9
|
||||
max_tokens: 4096
|
||||
timeout_seconds: 120
|
||||
```
|
||||
|
||||
Omit `context_compaction` to select automatically. Set it to `false` or `null` to disable it. The compactor must be a regular configured model group. The caller must have access to it and sufficient budget; configuring it grants no additional access
|
||||
|
||||
The input budget reserves the effective output allowance and a safety margin. Counts are estimates, so the provider remains authoritative. The router rechecks the compacted payload before sending it, attempts compaction only once per request, and reuses that attempt across answering retries. Compaction and answering are separate billed calls; output usage and therefore the final cost are not exactly knowable in advance
|
||||
|
||||
Responses compaction supports complete text history with ordinary function calls and results. Only its older prefix is converted for the compactor; retained items stay in their original format. Stored history (`previous_response_id` or `conversation`), explicit native context management, and opaque reasoning/compaction items remain on their existing client-managed path. This feature does not convert OpenAI encrypted state or claim it can be replayed across providers. The existing `/responses/compact` endpoint remains available for native OpenAI replay
|
||||
|
||||
A single oversized user turn, unclosed tool exchanges, unsupported Responses content, missing compactor capacity or an oversized summary produces a clear error instead of silently dropping context or changing the answerer. Fitting requests need no compaction. Models with unknown input limits retain their existing admission behavior
|
||||
|
||||
### Session pin retention
|
||||
|
||||
`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
|
|||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
|
||||
from litellm.router_strategy.complexity_router.tier_predictor import (
|
||||
TierSuccessPredictor,
|
||||
resolve_tier_artifact,
|
||||
|
|
@ -3257,7 +3258,11 @@ class ComplexityRouter(CustomLogger):
|
|||
resolved_messages: Sequence[Mapping[str, object]] | None,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> _RequestContextFit:
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
if (
|
||||
compaction_pending(request_kwargs)
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
|
||||
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
|
||||
(self.config.default_model,) if self.config.default_model else ()
|
||||
|
|
@ -3283,7 +3288,11 @@ class ComplexityRouter(CustomLogger):
|
|||
(the placement stands). Only a real tokenizer count ever moves a request, escalation
|
||||
lands only on groups whose every deployment declares a fitting window, and a group
|
||||
with no resolvable window is never moved on faith in either direction."""
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
if (
|
||||
compaction_pending(request_kwargs)
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
return None
|
||||
pools: Final = self._tier_pools()
|
||||
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))
|
||||
|
|
|
|||
|
|
@ -833,6 +833,15 @@ class CustomDimension(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ContextCompactionConfig(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
model: str | None = Field(default=None, min_length=1)
|
||||
trigger_ratio: float = Field(default=0.9, gt=0, lt=1)
|
||||
max_tokens: int = Field(default=4096, ge=512)
|
||||
timeout_seconds: float = Field(default=120, gt=0)
|
||||
|
||||
|
||||
class ComplexityRouterConfig(BaseModel):
|
||||
"""Configuration for the ComplexityRouter."""
|
||||
|
||||
|
|
@ -1304,6 +1313,16 @@ class ComplexityRouterConfig(BaseModel):
|
|||
),
|
||||
)
|
||||
|
||||
context_compaction: ContextCompactionConfig | Literal[False] = Field(
|
||||
default_factory=ContextCompactionConfig,
|
||||
description="Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.",
|
||||
)
|
||||
|
||||
@field_validator("context_compaction", mode="before")
|
||||
@classmethod
|
||||
def _normalize_context_compaction(cls, value: object) -> object:
|
||||
return False if value is None else value
|
||||
|
||||
enable_context_window_escalation: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
|
|
|
|||
395
litellm/router_strategy/complexity_router/context_compaction.py
Normal file
395
litellm/router_strategy/complexity_router/context_compaction.py
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from itertools import takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import parent_session_kwargs, sanitized_forwardable_call_metadata
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # legacy privacy owner accepts validated call details
|
||||
)
|
||||
from litellm.llms.anthropic import compaction as native
|
||||
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
Surface: TypeAlias = Literal["chat", "messages", "responses"]
|
||||
_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType(
|
||||
{"_acompletion": "chat", "anthropic_messages": "messages", "aresponses": "responses"}
|
||||
)
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_DICT: Final = TypeAdapter(dict[str, object])
|
||||
_ITEMS: Final = TypeAdapter(list[dict[str, object]])
|
||||
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_INPUT: Final = TypeAdapter[str | list[object] | None](str | list[object] | None)
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_STATE_KEY: Final = "_context_compaction_state"
|
||||
_native_child: Final[ContextVar[bool]] = ContextVar("native_compaction_child", default=False)
|
||||
|
||||
|
||||
class CompactionExecutor(Protocol):
|
||||
async def __call__(self, protocol: native.Protocol, payload: Mapping[str, object]) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
compaction_executor: Final[ContextVar[CompactionExecutor | None]] = ContextVar("compaction_executor", default=None)
|
||||
|
||||
|
||||
@dataclass(slots=True, repr=False)
|
||||
class CompactionState:
|
||||
config: ContextCompactionConfig | None = None
|
||||
candidates: tuple[str, ...] = ()
|
||||
summary: tuple[str, asyncio.Task[str]] | None = None
|
||||
|
||||
|
||||
def surface_for_call(function_name: str) -> Surface | None:
|
||||
return _SURFACES.get(function_name)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InputBudget:
|
||||
window: int | None
|
||||
available: int | None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def native_compaction_call() -> Generator[None]:
|
||||
token: Final = _native_child.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_native_child.reset(token)
|
||||
|
||||
|
||||
def initialize_compaction_state(kwargs: Mapping[str, object]) -> CompactionState:
|
||||
existing: Final = kwargs.get(_STATE_KEY)
|
||||
return existing if isinstance(existing, CompactionState) else CompactionState()
|
||||
|
||||
|
||||
def arm_compaction(
|
||||
kwargs: Mapping[str, object],
|
||||
config: ContextCompactionConfig | Literal[False] | None,
|
||||
candidates: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
state: Final = kwargs.get(_STATE_KEY)
|
||||
if isinstance(state, CompactionState):
|
||||
state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None
|
||||
state.candidates = candidates
|
||||
|
||||
|
||||
def _client_managed(payload: Mapping[str, object]) -> bool:
|
||||
return any(
|
||||
payload.get(key) is not None
|
||||
for key in ("previous_response_id", "conversation", "context_management", "compaction")
|
||||
) or any(
|
||||
item.get("type") in ("reasoning", "compaction", "item_reference")
|
||||
or item.get("encrypted_content") is not None
|
||||
or any(block.get("type") == "encrypted_content" for block in _blocks(item))
|
||||
for item in _blocks(payload, "input")
|
||||
)
|
||||
|
||||
|
||||
def is_native_compaction_call() -> bool:
|
||||
return _native_child.get()
|
||||
|
||||
|
||||
def reject_recursive_compactor(model: str) -> None:
|
||||
if _native_child.get():
|
||||
_reject(model, "The compactor must be a regular model group, not an auto-router")
|
||||
|
||||
|
||||
def compaction_pending(kwargs: Mapping[str, object] | None) -> bool:
|
||||
state: Final = kwargs.get(_STATE_KEY) if kwargs is not None else None
|
||||
return isinstance(state, CompactionState) and state.config is not None
|
||||
|
||||
|
||||
def _reject(model: str, reason: str) -> NoReturn:
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="")
|
||||
|
||||
|
||||
def _blocks(item: Mapping[str, object], key: str = "content") -> tuple[Mapping[str, object], ...]:
|
||||
value: Final = item.get(key)
|
||||
return _OBJECTS.validate_python(value) if isinstance(value, (list, tuple)) else ()
|
||||
|
||||
|
||||
def _tool_ids(items: Sequence[Mapping[str, object]], *, results: bool) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
identifier if isinstance(identifier, str) else ""
|
||||
for item in items
|
||||
for identifier in (
|
||||
*((item.get("tool_call_id"),) if results and item.get("role") == "tool" else ()),
|
||||
*(
|
||||
(item.get("call_id"),)
|
||||
if item.get("type") == ("function_call_output" if results else "function_call")
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
block.get("tool_use_id" if results else "id")
|
||||
for block in _blocks(item)
|
||||
if block.get("type") == ("tool_result" if results else "tool_use")
|
||||
),
|
||||
*(call.get("id") for call in _blocks(item, "tool_calls") if not results),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _history(
|
||||
items: Sequence[Mapping[str, object]], model: str
|
||||
) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]:
|
||||
instructions: Final = tuple(takewhile(lambda item: item.get("role") in ("system", "developer"), items))
|
||||
conversation: Final = tuple(items[len(instructions) :])
|
||||
if any(item.get("role") in ("system", "developer") for item in conversation):
|
||||
_reject(model, "Mid-conversation instructions cannot be compacted")
|
||||
split: Final = next(
|
||||
(
|
||||
index
|
||||
for index in range(len(conversation) - 1, -1, -1)
|
||||
if conversation[index].get("role") == "user"
|
||||
and not any(block.get("type") == "tool_result" for block in _blocks(conversation[index]))
|
||||
),
|
||||
0,
|
||||
)
|
||||
prefix: Final = conversation[:split]
|
||||
calls: Final = _tool_ids(prefix, results=False)
|
||||
results: Final = _tool_ids(prefix, results=True)
|
||||
if (
|
||||
not prefix
|
||||
or "" in calls
|
||||
or "" in results
|
||||
or len(calls) != len(frozenset(calls))
|
||||
or sorted(calls) != sorted(results)
|
||||
):
|
||||
_reject(model, "No closed older conversation is available without changing the latest request")
|
||||
return instructions, prefix, conversation[split:]
|
||||
|
||||
|
||||
async def _count(router: Router, payload: Mapping[str, object]) -> int:
|
||||
return await asyncio.to_thread(
|
||||
router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter
|
||||
messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None,
|
||||
input=_INPUT.validate_python(payload.get("input")),
|
||||
request_kwargs=payload,
|
||||
)
|
||||
|
||||
|
||||
def _budget(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
|
||||
) -> InputBudget:
|
||||
model: Final = str(payload.get("model", ""))
|
||||
info: Final = _MAPPING.validate_python(
|
||||
router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
|
||||
)
|
||||
raw_window: Final = info.get("max_input_tokens")
|
||||
window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None
|
||||
output: Final = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("max_completion_tokens", "max_output_tokens", "max_tokens")
|
||||
if payload.get(key) is not None
|
||||
),
|
||||
info.get("max_output_tokens"),
|
||||
)
|
||||
if output is not None and (not isinstance(output, int) or isinstance(output, bool) or output <= 0):
|
||||
_reject(model, "The output allowance must be a positive integer")
|
||||
return InputBudget(window, int(window * ratio) - output if window is not None and isinstance(output, int) else None)
|
||||
|
||||
|
||||
async def _compactor_model(router: Router, state: CompactionState, payload: Mapping[str, object]) -> str:
|
||||
needed: Final = await _count(router, payload)
|
||||
candidates: Final = (
|
||||
(state.config.model,) if state.config is not None and state.config.model is not None else state.candidates
|
||||
)
|
||||
selected: Final = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in candidates
|
||||
if (deployments := tuple(router.get_model_list(model_name=candidate) or ()))
|
||||
and all(
|
||||
native.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"]))
|
||||
and native.compatible_defaults(params)
|
||||
and (budget := _budget(router, deployment, payload, 0.9)).available is not None
|
||||
and needed <= budget.available
|
||||
for deployment in deployments
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
selected
|
||||
if selected is not None
|
||||
else _reject(
|
||||
str(payload["model"]),
|
||||
"No configured compactor supports native compaction with enough context and compatible defaults",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[str, object]:
|
||||
if surface != "responses":
|
||||
return payload
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
|
||||
|
||||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=TypeAdapter(ResponseInputParam).validate_python(payload["input"]),
|
||||
responses_api_request=_DICT.validate_python(payload),
|
||||
)
|
||||
tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
_OBJECTS.validate_python(payload.get("tools") or ())
|
||||
)
|
||||
return MappingProxyType({**payload, "messages": _ITEMS.validate_python(messages), "tools": tools})
|
||||
|
||||
|
||||
async def _generate_summary(
|
||||
router: Router, protocol: native.Protocol, payload: Mapping[str, object], timeout: float
|
||||
) -> str:
|
||||
executor: Final = compaction_executor.get()
|
||||
with native_compaction_call():
|
||||
response: Final = await asyncio.wait_for(
|
||||
executor(protocol, payload) if executor is not None else native.dispatch(router, protocol, payload),
|
||||
timeout=timeout,
|
||||
)
|
||||
summary: Final = native.extract_summary(protocol, response)
|
||||
return (
|
||||
summary
|
||||
if summary is not None
|
||||
else _reject(str(payload["model"]), "The provider did not return one complete native compaction block")
|
||||
)
|
||||
|
||||
|
||||
async def compact_to_fit(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
|
||||
) -> Mapping[str, object]:
|
||||
state: Final = payload.get(_STATE_KEY)
|
||||
config: Final = state.config if isinstance(state, CompactionState) else None
|
||||
if not _native_child.get() and (config is None or _client_managed(payload)):
|
||||
return payload
|
||||
model: Final = str(payload["model"])
|
||||
limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9)
|
||||
budget: Final = limits.available
|
||||
if budget is None or budget <= 0:
|
||||
if (
|
||||
config is not None
|
||||
and config.model is None
|
||||
and (limits.window is None or await _count(router, payload) <= limits.window)
|
||||
):
|
||||
return payload
|
||||
_reject(model, "A known input window and a smaller output allowance are required")
|
||||
if _native_child.get():
|
||||
if (
|
||||
not native.compatible_defaults(payload)
|
||||
or not native.supports_native_compaction(payload)
|
||||
or await _count(router, payload) > budget
|
||||
):
|
||||
_reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget")
|
||||
return payload
|
||||
if await _count(router, payload) <= budget:
|
||||
return payload
|
||||
if surface is None or config is None or not isinstance(state, CompactionState):
|
||||
_reject(model, "This request surface cannot be compacted")
|
||||
key: Final = "input" if surface == "responses" else "messages"
|
||||
raw_items: Final = payload[key]
|
||||
if isinstance(raw_items, str):
|
||||
_reject(model, "A single user input cannot be compacted without changing the latest request")
|
||||
items: Final = _ITEMS.validate_python(raw_items)
|
||||
if surface == "responses" and any(
|
||||
item.get("type", "message") not in ("message", "function_call", "function_call_output") for item in items
|
||||
):
|
||||
_reject(model, "Opaque or provider-managed Responses items require client-managed native compaction")
|
||||
if surface == "responses" and (
|
||||
any(block.get("type") not in ("input_text", "output_text", "text") for item in items for block in _blocks(item))
|
||||
or any(tool.get("type") != "function" for tool in _blocks(payload, "tools"))
|
||||
):
|
||||
_reject(model, "Only text history and ordinary function tools support portable Responses compaction")
|
||||
instructions, prefix, tail = _history(items, model)
|
||||
if any(
|
||||
item.get("thinking_blocks")
|
||||
or any(block.get("type") in ("thinking", "redacted_thinking", "compaction") for block in _blocks(item))
|
||||
for item in items
|
||||
):
|
||||
_reject(model, "Native reasoning or compaction blocks require client-managed native compaction")
|
||||
retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))})
|
||||
if await _count(router, retained) >= budget:
|
||||
_reject(model, "Retained instructions, tools and the latest turn leave no room for a summary")
|
||||
older: Final = _native_prefix(
|
||||
MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *prefix))}), surface
|
||||
)
|
||||
metadata: Final = sanitized_forwardable_call_metadata(
|
||||
_MAPPING.validate_python(payload.get("litellm_metadata") or payload.get("metadata") or _EMPTY),
|
||||
"autorouter_compaction",
|
||||
)
|
||||
protocol: Final[native.Protocol] = "messages" if surface == "messages" else "chat"
|
||||
request: Final = MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"messages": older["messages"],
|
||||
**native.request_kwargs(),
|
||||
"max_tokens": config.max_tokens,
|
||||
"stream": False,
|
||||
"num_retries": 0,
|
||||
"disable_fallbacks": True,
|
||||
"timeout": config.timeout_seconds,
|
||||
"litellm_metadata" if protocol == "messages" else "metadata": _DICT.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in metadata.items()
|
||||
if key != "user_api_key_auth" or compaction_executor.get() is None
|
||||
}
|
||||
)
|
||||
),
|
||||
**parent_session_kwargs(payload),
|
||||
**MappingProxyType({key: older[key] for key in ("system", "tools", "user") if key in older}),
|
||||
}
|
||||
)
|
||||
compactor: Final = await _compactor_model(router, state, request)
|
||||
child: Final = MappingProxyType({**request, "model": compactor})
|
||||
identity: Final = hashlib.sha256(
|
||||
json.dumps(
|
||||
(protocol, compactor, child["messages"], child.get("system"), child.get("tools")), sort_keys=True
|
||||
).encode()
|
||||
).hexdigest()
|
||||
if state.summary is None:
|
||||
private: Final = should_redact_message_logging(
|
||||
_DICT.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
"litellm_params": payload,
|
||||
"standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(
|
||||
_DICT.validate_python(payload)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
with inherit_message_logging_privacy(private):
|
||||
state.summary = (
|
||||
identity,
|
||||
asyncio.create_task(_generate_summary(router, protocol, child, config.timeout_seconds)),
|
||||
)
|
||||
if state.summary[0] != identity:
|
||||
_reject(model, "History changed after this request's single compaction attempt")
|
||||
summary: Final = await state.summary[1]
|
||||
message: Final = MappingProxyType(
|
||||
{"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary}
|
||||
)
|
||||
compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))})
|
||||
if await _count(router, compacted) > budget:
|
||||
_reject(model, "The summary and retained conversation still exceed the selected deployment's budget")
|
||||
return compacted
|
||||
|
|
@ -24,7 +24,7 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
|
|||
|
||||
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
|
||||
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding"]
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "compactor"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -154,6 +154,7 @@ def strategy_router_dependencies(
|
|||
dict.fromkeys(
|
||||
tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier"))
|
||||
+ _named(litellm_params.get("complexity_router_default_model"), "default")
|
||||
+ _named(_mapping(complexity.get("context_compaction")).get("model"), "compactor")
|
||||
+ (
|
||||
_named(classifier.get("model"), "classifier")
|
||||
if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES
|
||||
|
|
|
|||
|
|
@ -216,11 +216,20 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False):
|
|||
caller: ToolCaller | None
|
||||
|
||||
|
||||
class CompactionBlock(TypedDict, total=False):
|
||||
"""Native compaction block, signed for on-demand compaction."""
|
||||
|
||||
type: Required[ReadOnly[Literal["compaction"]]]
|
||||
content: ReadOnly[str | None]
|
||||
signature: ReadOnly[str]
|
||||
|
||||
|
||||
AnthropicMessagesAssistantMessageValues = (
|
||||
AnthropicMessagesTextParam
|
||||
| AnthropicMessagesToolUseParam
|
||||
| ChatCompletionThinkingBlock
|
||||
| ChatCompletionRedactedThinkingBlock
|
||||
| CompactionBlock
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -390,6 +399,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = (
|
|||
)
|
||||
|
||||
|
||||
class AnthropicCompaction(TypedDict, total=False):
|
||||
type: Required[ReadOnly[Literal["summarize"]]]
|
||||
instructions: ReadOnly[str]
|
||||
|
||||
|
||||
class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
||||
max_tokens: int | None
|
||||
metadata: AnthropicMetadata | dict | None
|
||||
|
|
@ -405,6 +419,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
|||
top_p: float | None
|
||||
mcp_servers: list[AnthropicMcpServerTool] | None
|
||||
context_management: dict[str, Any] | None
|
||||
compaction: ReadOnly[AnthropicCompaction | None]
|
||||
container: dict[str, Any] | None # Container config with skills for code execution
|
||||
output_format: AnthropicOutputSchema | None # Structured outputs support
|
||||
speed: str | None # Fast mode support for Opus models
|
||||
|
|
@ -564,13 +579,6 @@ class ContextManagementResponse(TypedDict, total=False):
|
|||
applied_edits: list[AppliedEdit]
|
||||
|
||||
|
||||
class CompactionBlock(TypedDict, total=False):
|
||||
"""Synthesized ``compaction`` content block (compact_20260112)."""
|
||||
|
||||
type: Required[Literal["compaction"]]
|
||||
content: str | None
|
||||
|
||||
|
||||
class UsageIteration(TypedDict, total=False):
|
||||
"""One sampling iteration's token usage (compact_20260112)."""
|
||||
|
||||
|
|
@ -743,6 +751,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
WEB_SEARCH_2025_03_05 = "web-search-2025-03-05"
|
||||
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
|
||||
COMPACT_2026_01_12 = "compact-2026-01-12"
|
||||
COMPACT_2026_09_04 = "compact-2026-09-04"
|
||||
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
|
||||
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
|
||||
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Sequence
|
||||
from typing import Any, Literal, TypeAlias
|
||||
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
|
@ -6,8 +7,10 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicResponseContentBlockText,
|
||||
AnthropicResponseContentBlockToolUse,
|
||||
AnthropicStopDetails,
|
||||
CompactionBlock,
|
||||
ContextManagementResponse,
|
||||
ServerToolUsage,
|
||||
UsageIteration,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -56,6 +59,7 @@ AnthropicResponseContentBlock: TypeAlias = (
|
|||
| AnthropicResponseToolUseBlock
|
||||
| AnthropicResponseThinkingBlock
|
||||
| AnthropicResponseRedactedThinkingBlock
|
||||
| CompactionBlock
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -66,6 +70,7 @@ class AnthropicUsage(TypedDict, total=False):
|
|||
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
iterations: ReadOnly[Sequence[UsageIteration]]
|
||||
|
||||
"""
|
||||
Cache Tokens Used
|
||||
|
|
@ -91,7 +96,9 @@ class AnthropicMessagesResponse(TypedDict, total=False):
|
|||
id: str
|
||||
model: str | None # This represents the Model type from Anthropic
|
||||
role: Literal["assistant"] | None
|
||||
stop_reason: Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] | None
|
||||
stop_reason: ReadOnly[
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
|
||||
]
|
||||
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
|
||||
stop_sequence: str | None
|
||||
type: Literal["message"] | None
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_output_config: bool | None
|
||||
supports_image_size: bool | None
|
||||
supports_anthropic_thinking_payload: ReadOnly[bool | None]
|
||||
supports_anthropic_compaction: ReadOnly[bool | None]
|
||||
supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None]
|
||||
vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None]
|
||||
bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None
|
||||
|
|
@ -2960,6 +2961,7 @@ RoutingDecisionCause = Literal[
|
|||
|
||||
InternalCallOrigin = Literal[
|
||||
"autorouter_classifier",
|
||||
"autorouter_compaction",
|
||||
"shadow_eval_router",
|
||||
"shadow_eval_judge",
|
||||
"llm_as_a_judge_guardrail",
|
||||
|
|
@ -3835,6 +3837,7 @@ all_litellm_params = (
|
|||
agentic_loop_internal_litellm_params
|
||||
+ [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params]
|
||||
+ [
|
||||
"_context_compaction_state",
|
||||
"metadata",
|
||||
"litellm_metadata",
|
||||
"keepalive_seconds",
|
||||
|
|
|
|||
|
|
@ -6120,6 +6120,7 @@ def _get_model_info_helper(
|
|||
supports_tool_search=_model_info.get("supports_tool_search", None),
|
||||
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
|
||||
supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None),
|
||||
supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None),
|
||||
supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),
|
||||
supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None),
|
||||
supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),
|
||||
|
|
|
|||
|
|
@ -14487,6 +14487,7 @@
|
|||
"source": "https://docs.anthropic.com/en/docs/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 4e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -14526,6 +14527,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-sonnet-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 6e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
|
|
@ -14741,6 +14743,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14779,6 +14782,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-6-20260205": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14816,6 +14820,7 @@
|
|||
"prompt_cache_min_tokens": 4096
|
||||
},
|
||||
"claude-opus-4-7": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14855,6 +14860,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-7-20260416": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -14893,6 +14899,7 @@
|
|||
"prompt_cache_min_tokens": 2048
|
||||
},
|
||||
"claude-fable-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -14932,6 +14939,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-fable-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -14972,6 +14980,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -15014,6 +15023,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-opus-4-8": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -60334,6 +60344,7 @@
|
|||
"supports_audio_output": true
|
||||
},
|
||||
"claude-mythos-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
@ -60373,6 +60384,7 @@
|
|||
}
|
||||
},
|
||||
"claude-mythos-5-1": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -60413,6 +60425,7 @@
|
|||
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
|
||||
},
|
||||
"claude-mythos-preview": {
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
|
|
|
|||
|
|
@ -815,6 +815,9 @@
|
|||
"supports_adaptive_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_anthropic_compaction": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_anthropic_thinking_payload": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ ALLOWED_FILES = [
|
|||
"../../litellm/llms/custom_httpx/httpx_handler.py",
|
||||
"../../litellm/llms/openai/common_utils.py",
|
||||
"../../litellm/experimental_mcp_client/client.py",
|
||||
"../../litellm/proxy/native_compaction.py",
|
||||
# when running on ci/cd
|
||||
"./litellm/__init__.py",
|
||||
"./litellm/llms/custom_httpx/http_handler.py",
|
||||
|
|
@ -22,6 +23,7 @@ ALLOWED_FILES = [
|
|||
"./litellm/llms/custom_httpx/httpx_handler.py",
|
||||
"./litellm/llms/openai/common_utils.py",
|
||||
"./litellm/experimental_mcp_client/client.py",
|
||||
"./litellm/proxy/native_compaction.py",
|
||||
]
|
||||
|
||||
warning_msg = "this is a serious violation that can impact latency. Creating Async clients per request can add +500ms per request"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
iter_client_callback_metadata_dicts,
|
||||
)
|
||||
|
|
@ -189,6 +193,20 @@ def test_empty_kwargs_returns_empty_params():
|
|||
assert dict(params) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("child_privacy", (False, True))
|
||||
def test_inherited_privacy_only_strengthens_child_and_resets(child_privacy: bool) -> None:
|
||||
kwargs: Final = TypeAdapter(dict[str, object]).validate_python(
|
||||
MappingProxyType({"turn_off_message_logging": child_privacy})
|
||||
)
|
||||
with inherit_message_logging_privacy(False):
|
||||
assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy
|
||||
with inherit_message_logging_privacy(True), inherit_message_logging_privacy(False):
|
||||
params: Final = initialize_standard_callback_dynamic_params(kwargs)
|
||||
assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy
|
||||
assert params["turn_off_message_logging"] is True
|
||||
assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None
|
||||
|
||||
|
||||
def test_newrelic_callback_params_are_not_extracted_from_request_kwargs():
|
||||
kwargs = {
|
||||
"newrelic_api_key": "caller-key",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
|
||||
import pytest
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
|
|
@ -3771,6 +3775,69 @@ def test_multiple_compaction_blocks():
|
|||
assert compaction_blocks[1]["content"] == "Second summary..."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("messages_api,gateway,native_endpoint", [
|
||||
(False, False, False), (True, False, False), (False, True, False), (True, True, False), (True, True, True),
|
||||
])
|
||||
async def test_native_compaction_wire_roundtrip(
|
||||
messages_api: bool, gateway: bool, native_endpoint: bool,
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
|
||||
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
|
||||
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False)
|
||||
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
|
||||
operation: Final = {"type": "summarize", "instructions": "Keep identifiers"}
|
||||
usage: Final = {"input_tokens": 0, "output_tokens": 0,
|
||||
"iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}]}
|
||||
chat_wire: Final = gateway and not native_endpoint
|
||||
base: Final = "https://gateway.test/v1" if gateway else "https://api.anthropic.com/v1"
|
||||
route: Final = respx_mock.post(f"{base}/{'chat/completions' if chat_wire else 'messages'}")
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert len(request.headers.get_list("anthropic-beta")) == 1
|
||||
assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == {
|
||||
"compact-2026-09-04", "interleaved-thinking-2025-05-14",
|
||||
}
|
||||
if "compaction" in payload:
|
||||
assert payload["compaction"] == operation
|
||||
else:
|
||||
assert payload["messages"][0] == {"role": "assistant", "content": [block]}
|
||||
body: Final = (
|
||||
{"id": "chatcmpl_compact", "object": "chat.completion", "created": 1, "model": "claude-sonnet-5",
|
||||
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "",
|
||||
"provider_specific_fields": {"compaction_blocks": [block]}}}],
|
||||
"usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268}}
|
||||
if chat_wire else
|
||||
{"id": "msg_compact", "type": "message", "role": "assistant", "model": "claude-sonnet-5",
|
||||
"content": [block], "stop_reason": "compaction", "usage": usage}
|
||||
)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
route.mock(side_effect=respond)
|
||||
call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion
|
||||
params: Final = dict(
|
||||
model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", max_tokens=512,
|
||||
api_base=base if gateway else "https://api.anthropic.com",
|
||||
extra_headers={"Anthropic-Beta": f"interleaved-thinking-2025-05-14{',compact-2026-09-04' if gateway else ''}"},
|
||||
model_info={"supported_endpoints": ["/v1/messages"]} if native_endpoint else {},
|
||||
)
|
||||
response: Final = await call(
|
||||
messages=[{"role": "user", "content": "Remember identifiers"}], compaction=operation, **params
|
||||
)
|
||||
message: Final = response if messages_api else response.choices[0].message.model_dump()
|
||||
blocks: Final = message["content"] if messages_api else message["provider_specific_fields"]["compaction_blocks"]
|
||||
assert blocks == [block]
|
||||
if messages_api:
|
||||
assert response["stop_reason"] == "compaction"
|
||||
if not chat_wire:
|
||||
assert response["usage"] == usage
|
||||
if not gateway:
|
||||
replay: Final = {"role": "assistant", "content": blocks} if messages_api else message
|
||||
await call(messages=[replay, {"role": "user", "content": "Continue"}], **params)
|
||||
assert route.call_count == (1 if gateway else 2)
|
||||
|
||||
|
||||
def test_compaction_block_request_transformation():
|
||||
"""
|
||||
Test that compaction blocks from provider_specific_fields are correctly
|
||||
|
|
|
|||
|
|
@ -60,6 +60,20 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None:
|
|||
assert result["usage"]["input_tokens"] == 10
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text,count,expected_stop", [
|
||||
("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"),
|
||||
(" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"),
|
||||
])
|
||||
def test_native_compaction_response_roundtrip(text: str | None, count: int, expected_stop: str) -> None:
|
||||
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
|
||||
message: Final = Message(content=text, provider_specific_fields={"compaction_blocks": [block] * count})
|
||||
response: Final = ModelResponse(choices=[Choices(message=message, finish_reason="length")], usage=Usage())
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
|
||||
expected_text: Final = [{"type": "text", "text": text}] if text is not None and (text != "" or not count) else []
|
||||
assert result["content"] == [*([block] * count), *expected_text]
|
||||
assert result["stop_reason"] == expected_stop
|
||||
|
||||
|
||||
def test_translate_chat_refusal_to_anthropic_response():
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-refusal",
|
||||
|
|
|
|||
145
tests/test_litellm/proxy/test_native_compaction.py
Normal file
145
tests/test_litellm/proxy/test_native_compaction.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import asyncio
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, Request
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.proxy import common_request_processing, proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash, get_request_stash
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_executor, reject_recursive_compactor
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
_HEADERS: Final = (
|
||||
(b"authorization", b"Bearer sk-compaction-fixture"), (b"cookie", b"session=fixture"),
|
||||
(b"content-length", b"99999"), (b"x-litellm-call-id", b"parent"),
|
||||
(b"litellm-disable-message-redaction", b"true"), (b"x-litellm-num-retries", b"8"),
|
||||
(b"X-LiteLLM-Timeout", b"600"), (b"x-litellm-stream-timeout", b"500"),
|
||||
)
|
||||
|
||||
|
||||
async def _child(protocol: Literal["chat", "messages"] = "chat", forged: bool = False) -> Mapping[str, object]:
|
||||
executor: Final = compaction_executor.get()
|
||||
assert executor is not None
|
||||
payload: Final = TypeAdapter(Mapping[str, object]).validate_json(
|
||||
b'{"model":"compactor","messages":[{"role":"user","content":"history"}],'
|
||||
b'"num_retries":0,"timeout":7,"stream_timeout":7,"disable_fallbacks":true,"stream":false,'
|
||||
b'"metadata":{"turn_off_message_logging":true}}'
|
||||
)
|
||||
return await executor(protocol, MappingProxyType({
|
||||
"litellm_metadata" if protocol == "messages" and key == "metadata" else key: value
|
||||
for key, value in payload.items() if forged or key != "metadata"
|
||||
}))
|
||||
|
||||
|
||||
def _request(app: FastAPI) -> Request:
|
||||
return Request(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({
|
||||
"type": "http", "app": app, "scheme": "https", "server": ("proxy.test", 443),
|
||||
"path": "/gateway/parent", "root_path": "/gateway", "query_string": b"parent=1",
|
||||
"client": ("192.0.2.1", 4321), "headers": _HEADERS,
|
||||
})))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("chat", "messages"))
|
||||
async def test_child_preserves_credentials_and_isolates_context(protocol: Literal["chat", "messages"]) -> None:
|
||||
app: Final = FastAPI()
|
||||
stash: Final = get_or_create_request_stash()
|
||||
|
||||
@app.post("/v1/chat/completions" if protocol == "chat" else "/v1/messages")
|
||||
async def endpoint(request: Request) -> Mapping[str, object]:
|
||||
assert get_request_stash() is None and compaction_executor.get() is None
|
||||
assert request.client == ("192.0.2.1", 4321) and request.url.scheme == "https"
|
||||
assert request.scope["root_path"] == "/gateway" and request.cookies["session"] == "fixture"
|
||||
assert request.headers["authorization"] == "Bearer sk-compaction-fixture" and not request.query_params
|
||||
assert "x-litellm-call-id" not in request.headers
|
||||
assert "litellm-disable-message-redaction" not in request.headers
|
||||
assert int(request.headers["content-length"]) == len(await request.body())
|
||||
with pytest.raises(BadRequestError, match="regular model group"):
|
||||
reject_recursive_compactor("auto-router")
|
||||
return MappingProxyType({"summary": "compacted"})
|
||||
|
||||
with inherit_message_logging_privacy(True):
|
||||
assert (await with_proxy_compaction_executor(_child(protocol), _request(app)))["summary"] == "compacted"
|
||||
assert get_request_stash() is stash and compaction_executor.get() is None
|
||||
reject_recursive_compactor("auto-router")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("chat", "messages"))
|
||||
@pytest.mark.parametrize("policy", ("allowed", "denied", "forged"))
|
||||
async def test_real_proxy_child_auth_privacy_and_body_policy(
|
||||
monkeypatch: pytest.MonkeyPatch, protocol: Literal["chat", "messages"], policy: str,
|
||||
) -> None:
|
||||
cache: Final = DualCache()
|
||||
token: Final = proxy_server.hash_token("sk-compaction-fixture")
|
||||
await cache.async_set_cache(key=token, value=UserAPIKeyAuth.model_validate(
|
||||
MappingProxyType({"token": token, "models": ("answer",) if policy == "denied" else ("compactor",)})
|
||||
))
|
||||
dispatched: Final = asyncio.Event()
|
||||
|
||||
async def route(
|
||||
data: Mapping[str, object], llm_router: Router | None, user_model: str | None,
|
||||
route_type: str, user_api_key_dict: UserAPIKeyAuth | None,
|
||||
) -> Awaitable[ModelResponse]:
|
||||
dispatched.set()
|
||||
assert policy == "allowed"
|
||||
assert (data["num_retries"], data["timeout"], data["stream_timeout"]) == (0, 7, 7)
|
||||
assert data["disable_fallbacks"] is True and data["stream"] is False
|
||||
logging: Final = data["litellm_logging_obj"]
|
||||
assert isinstance(logging, Logging)
|
||||
assert logging.standard_callback_dynamic_params.get("turn_off_message_logging") is True
|
||||
assert should_redact_message_logging(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({
|
||||
"litellm_params": data, "standard_callback_dynamic_params": logging.standard_callback_dynamic_params,
|
||||
})))
|
||||
return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor"))
|
||||
|
||||
monkeypatch.setattr(proxy_server.app, "dependency_overrides", {})
|
||||
monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture")
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", object())
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", cache)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(common_request_processing, "route_request", route)
|
||||
with inherit_message_logging_privacy(True):
|
||||
call: Final = with_proxy_compaction_executor(_child(protocol, policy == "forged"), _request(proxy_server.app))
|
||||
if policy == "allowed":
|
||||
assert (await call)["id"] == "private-summary"
|
||||
else:
|
||||
status: Final = 403 if policy == "denied" else 401
|
||||
with pytest.raises(BadRequestError, match=rf"child request failed \(HTTP {status}\)"):
|
||||
await call
|
||||
assert dispatched.is_set() is (policy == "allowed")
|
||||
assert compaction_executor.get() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelling_parent_cancels_and_drains_child() -> None:
|
||||
app: Final = FastAPI()
|
||||
started: Final = asyncio.Event()
|
||||
stopped: Final = asyncio.Event()
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def endpoint() -> None:
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
stopped.set()
|
||||
|
||||
parent: Final = asyncio.create_task(with_proxy_compaction_executor(_child(), _request(app)))
|
||||
await asyncio.wait_for(started.wait(), timeout=5)
|
||||
parent.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await parent
|
||||
assert stopped.is_set() and compaction_executor.get() is None
|
||||
|
|
@ -0,0 +1,393 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping
|
||||
from copy import deepcopy
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic import compaction as native
|
||||
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
CompactionState,
|
||||
Surface,
|
||||
arm_compaction,
|
||||
compact_to_fit,
|
||||
compaction_executor,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
pytestmark: Final = [pytest.mark.asyncio, pytest.mark.usefixtures("local_model_cost_map")]
|
||||
SCHEMA: Final = {"type": "object", "properties": {"code": {"type": "string"}}}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def native_catalog(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> None:
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setenv("LITELLM_LICENSE", "")
|
||||
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False)
|
||||
monkeypatch.setitem(litellm.model_cost, "summary-fixture", {
|
||||
"litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096, "supports_anthropic_compaction": True,
|
||||
})
|
||||
|
||||
|
||||
def make_router(
|
||||
window: int | None = 512, settings: Mapping[str, object] | None = None, *, compactor_window: int = 32000,
|
||||
conflict: bool = False, output: int | None = 64,
|
||||
) -> litellm.Router:
|
||||
config: Final = {
|
||||
"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"},
|
||||
"keyword_tier_rules": [{"keywords": ["answer", "tail result"], "tier": "SIMPLE"}],
|
||||
"enable_context_window_escalation": False, "max_tokens_from_tier_model": False,
|
||||
**(settings or {}),
|
||||
}
|
||||
return litellm.Router(model_list=[
|
||||
{"model_name": "auto", "litellm_params": {
|
||||
"model": "auto_router/complexity_router", "complexity_router_config": config,
|
||||
}},
|
||||
{"model_name": "small", "litellm_params": {
|
||||
"model": "openai/arbitrary-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", "max_retries": 0,
|
||||
}, "model_info": {"id": "pinned-answer", "max_input_tokens": window, "max_output_tokens": output}},
|
||||
{"model_name": "large", "litellm_params": {
|
||||
"model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "compact-test",
|
||||
**({"stop": ["deployment policy"]} if conflict else {}),
|
||||
}, "model_info": {"id": "native-compactor", "max_input_tokens": compactor_window, "max_output_tokens": 4096}},
|
||||
{"model_name": "backup", "litellm_params": {
|
||||
"model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "backup-test",
|
||||
}, "model_info": {"id": "backup-compactor", "max_input_tokens": 32000, "max_output_tokens": 4096}},
|
||||
], enable_pre_call_checks=True, num_retries=0, disable_cooldowns=True,
|
||||
retry_policy={"InternalServerErrorRetries": 1})
|
||||
|
||||
|
||||
def exchange(surface: Surface, phase: str) -> list[dict[str, object]]:
|
||||
identifier: Final = f"{phase}-call"
|
||||
result: Final = f"{phase} result"
|
||||
if surface == "responses":
|
||||
return [
|
||||
{"type": "function_call", "call_id": identifier, "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": identifier, "output": result},
|
||||
]
|
||||
if surface == "messages":
|
||||
return [
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": identifier, "name": "lookup", "input": {}}]},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": identifier, "content": result}]},
|
||||
]
|
||||
return [
|
||||
{"role": "assistant", "tool_calls": [
|
||||
{"id": identifier, "type": "function", "function": {"name": "lookup", "arguments": "{}"}},
|
||||
]},
|
||||
{"role": "tool", "tool_call_id": identifier, "content": result},
|
||||
]
|
||||
|
||||
|
||||
def history(surface: Surface) -> dict[str, object]:
|
||||
conversation: Final = [
|
||||
{"role": "user", "content": "Project code MAPLE-47. Background detail. " * 150},
|
||||
{"role": "assistant", "content": "Recorded"},
|
||||
*exchange(surface, "prefix"),
|
||||
{"role": "user", "content": "Answer with the project code"},
|
||||
*exchange(surface, "tail"),
|
||||
]
|
||||
function: Final = {"name": "lookup", "parameters": SCHEMA}
|
||||
if surface == "responses":
|
||||
return {"instructions": "Keep the code exact", "tools": [{"type": "function", **function}], "input": [
|
||||
{"role": "developer", "content": "Retain the original spelling"}, *conversation,
|
||||
]}
|
||||
if surface == "messages":
|
||||
return {"system": "Keep the code exact", "tools": [{"name": "lookup", "input_schema": SCHEMA}], "messages": [
|
||||
*conversation,
|
||||
]}
|
||||
return {"tools": [{"type": "function", "function": function}], "messages": [
|
||||
{"role": "system", "content": "Keep the code exact"}, *conversation,
|
||||
]}
|
||||
|
||||
|
||||
def native_reply(summary: str = "Project code MAPLE-47", signed: bool = True, truncated: bool = False) -> httpx.Response:
|
||||
return httpx.Response(200, json={
|
||||
"id": "msg_compact", "type": "message", "role": "assistant", "model": "summary-fixture",
|
||||
"content": [{"type": "compaction", "content": summary, **({"signature": "native-signature"} if signed else {})}],
|
||||
"stop_reason": "max_tokens" if truncated else "compaction", "usage": {"input_tokens": 0, "output_tokens": 0, "iterations": [
|
||||
{"type": "compaction", "input_tokens": 1200, "output_tokens": 20},
|
||||
]},
|
||||
})
|
||||
|
||||
|
||||
def answer_reply(request: httpx.Request) -> httpx.Response:
|
||||
payload: Final = json.loads(request.content)
|
||||
assert payload["model"] == "arbitrary-answer"
|
||||
assert request.headers["authorization"] == "Bearer answer-test"
|
||||
if request.url.path.endswith("responses"):
|
||||
return httpx.Response(200, json={
|
||||
"id": "resp_answer", "object": "response", "created_at": 0, "status": "completed",
|
||||
"model": payload["model"], "output": [{"id": "msg_answer", "type": "message", "role": "assistant",
|
||||
"status": "completed", "content": [{"type": "output_text", "text": "MAPLE-47", "annotations": []}]}],
|
||||
"usage": {"input_tokens": 60, "output_tokens": 8, "total_tokens": 68},
|
||||
})
|
||||
return httpx.Response(200, json={
|
||||
"id": "answer", "object": "chat.completion", "created": 0, "model": payload["model"],
|
||||
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "MAPLE-47"}}],
|
||||
"usage": {"prompt_tokens": 60, "completion_tokens": 8, "total_tokens": 68},
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wire() -> Iterator[tuple[respx.Route, respx.Route]]:
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
compactor: Final = transport.post("https://compact.test/v1/messages").mock(return_value=native_reply())
|
||||
answer: Final = transport.route(method="POST", host="answer.test").mock(side_effect=answer_reply)
|
||||
yield compactor, answer
|
||||
|
||||
|
||||
async def invoke(router: litellm.Router, surface: Surface, payload: Mapping[str, object], retries: int = 0) -> object:
|
||||
if surface == "responses":
|
||||
return await router.aresponses(model="auto", max_output_tokens=64, num_retries=retries, **payload)
|
||||
if surface == "messages":
|
||||
return await router.aanthropic_messages(model="auto", max_tokens=64, num_retries=retries, **payload)
|
||||
return await router.acompletion(model="auto", max_tokens=64, num_retries=retries, **payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("surface", ["chat", "messages", "responses"])
|
||||
@pytest.mark.parametrize("near", [False, True])
|
||||
@pytest.mark.parametrize("configured", [False, True])
|
||||
async def test_all_surfaces_compact_and_keep_selected_answerer(
|
||||
wire: tuple[respx.Route, respx.Route], surface: Surface, near: bool, configured: bool,
|
||||
) -> None:
|
||||
payload: Final = history(surface)
|
||||
original: Final = deepcopy(payload)
|
||||
counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload)
|
||||
window: Final = int((counted + 32) / ContextCompactionConfig().trigger_ratio) + 1 if near else 512
|
||||
assert (counted < window) is near
|
||||
settings: Final = {"context_compaction": {"model": "large", "max_tokens": 512}} if configured else {}
|
||||
router: Final = make_router(window, settings)
|
||||
captured: Final = asyncio.Queue[Mapping[str, object]]()
|
||||
compactor, answer = wire
|
||||
retry: Final = near and configured
|
||||
|
||||
def answer_after_retry(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(500, json={"error": {"message": "retry answer"}}) if answer.call_count == 0 else answer_reply(request)
|
||||
|
||||
if retry:
|
||||
answer.mock(side_effect=answer_after_retry)
|
||||
|
||||
async def execute(protocol: native.Protocol, request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
result: Final = await native.dispatch(router, protocol, request)
|
||||
captured.put_nowait(result)
|
||||
return result
|
||||
|
||||
token: Final = compaction_executor.set(execute)
|
||||
try:
|
||||
response: Final = await invoke(router, surface, payload, retries=int(retry))
|
||||
finally:
|
||||
compaction_executor.reset(token)
|
||||
assert compactor.call_count == captured.qsize() == 1
|
||||
assert answer.call_count == router.total_calls["openai/arbitrary-answer"] == 1 + int(retry)
|
||||
compact_request: Final = compactor.calls[0].request
|
||||
compact_body: Final = json.loads(compact_request.content)
|
||||
answer_body: Final = json.loads(answer.calls[0].request.content)
|
||||
assert answer_body == json.loads(answer.calls[-1].request.content)
|
||||
assert compact_body["model"] == "summary-fixture" and compact_body["compaction"] == {"type": "summarize"}
|
||||
assert ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value in compact_request.headers["anthropic-beta"].split(",")
|
||||
assert compact_body["max_tokens"] == (512 if configured else ContextCompactionConfig().max_tokens)
|
||||
assert "Background detail" in str(compact_body) and "tail-call" not in str(compact_body)
|
||||
assert "prefix-call" in str(compact_body) and "prefix result" in str(compact_body)
|
||||
assert "Keep the code exact" in str(compact_body["system"])
|
||||
assert compact_body["tools"][0]["input_schema"] == SCHEMA
|
||||
assert "MAPLE-47" in str(answer_body) and "Background detail" not in str(answer_body)
|
||||
assert "prefix-call" not in str(answer_body)
|
||||
assert "native-signature" not in str(answer_body) and "compaction" not in answer_body
|
||||
assert "tail-call" in str(answer_body) and "tail result" in str(answer_body)
|
||||
assert "MAPLE-47" in str(response)
|
||||
usage: Final = captured.get_nowait()["usage"]
|
||||
if surface == "messages":
|
||||
assert usage["iterations"][0]["input_tokens"] == 1200 and usage["iterations"][0]["output_tokens"] == 20
|
||||
else:
|
||||
assert usage["prompt_tokens"] == 1200 and usage["completion_tokens"] == 20
|
||||
if surface == "responses":
|
||||
assert answer_body["input"][-3:] == original["input"][-3:]
|
||||
assert answer_body["input"][0] == original["input"][0]
|
||||
assert answer_body["instructions"] == original["instructions"] and answer_body["tools"] == original["tools"]
|
||||
assert payload == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("surface", ["chat", "messages", "responses"])
|
||||
@pytest.mark.parametrize("mode", ["fitting", "false", "null"])
|
||||
async def test_fitting_and_disabled_requests_do_not_compact(
|
||||
wire: tuple[respx.Route, respx.Route], surface: Surface, mode: str,
|
||||
) -> None:
|
||||
settings: Final = {} if mode == "fitting" else {"context_compaction": False if mode == "false" else None}
|
||||
payload: Final = history(surface)
|
||||
original: Final = deepcopy(payload)
|
||||
counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload)
|
||||
await invoke(make_router(20000 if mode == "fitting" else counted + 64, settings), surface, payload)
|
||||
compactor, answer = wire
|
||||
assert compactor.call_count == 0 and answer.call_count == 1
|
||||
assert "Background detail" in answer.calls[0].request.content.decode()
|
||||
assert payload == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("surface", "failure"),
|
||||
[(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")]
|
||||
+ [("messages", "truncated")],
|
||||
)
|
||||
async def test_bad_native_result_never_reaches_answerer(
|
||||
wire: tuple[respx.Route, respx.Route], surface: Surface, failure: str,
|
||||
) -> None:
|
||||
compactor, answer = wire
|
||||
reply: Final = httpx.Response(500, json={"error": {"type": "api_error", "message": "failed"}}) if failure == "provider" else native_reply(
|
||||
"too large " * 2000 if failure == "oversized" else "MAPLE-47", signed=failure != "unsigned",
|
||||
truncated=failure == "truncated",
|
||||
)
|
||||
compactor.mock(return_value=reply)
|
||||
with pytest.raises((litellm.BadRequestError, litellm.InternalServerError)):
|
||||
await invoke(make_router(), surface, history(surface))
|
||||
assert compactor.call_count == 1 and answer.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["item", "content", "tools", "unclosed", "missing", "duplicate", "instructions", "retained"])
|
||||
@pytest.mark.parametrize("needed", [False, True])
|
||||
async def test_unsafe_responses_reject_only_when_compaction_needed(
|
||||
wire: tuple[respx.Route, respx.Route], failure: str, needed: bool,
|
||||
) -> None:
|
||||
payload: Final = history("responses")
|
||||
extra: Final = {
|
||||
"item": [{"type": "computer_call", "call_id": "opaque-tool"}],
|
||||
"content": [{"role": "assistant", "content": [{"type": "refusal", "refusal": "cannot"}]}],
|
||||
"unclosed": [{"type": "function_call", "call_id": "unclosed", "name": "lookup", "arguments": "{}"}],
|
||||
"missing": [{"type": "function_call", "name": "lookup", "arguments": "{}"}],
|
||||
"duplicate": exchange("responses", "prefix"),
|
||||
"instructions": [{"role": "developer", "content": "Changed instructions"}],
|
||||
}
|
||||
request: Final = {
|
||||
**payload, "input": [*payload["input"][:3], *extra.get(failure, []), *payload["input"][3:]],
|
||||
**({"tools": [{"type": "computer_use_preview", "display_width": 800, "display_height": 600}]} if failure == "tools" else {}),
|
||||
**({"instructions": "Keep every instruction " * 600} if failure == "retained" else {}),
|
||||
}
|
||||
if needed:
|
||||
with pytest.raises(litellm.BadRequestError, match="Context compaction"):
|
||||
await invoke(make_router(), "responses", request)
|
||||
assert all(route.call_count == 0 for route in wire)
|
||||
else:
|
||||
await invoke(make_router(20000), "responses", request)
|
||||
compactor, answer = wire
|
||||
assert compactor.call_count == 0 and answer.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("owned", [
|
||||
{"previous_response_id": "resp_parent"}, {"conversation": "conv_parent"},
|
||||
{"context_management": [{"type": "compaction", "compact_threshold": 1000}]}, {"compaction": {"type": "summarize"}},
|
||||
{"input": [{"type": "reasoning", "encrypted_content": "opaque"}]},
|
||||
{"input": [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "prior reasoning"}]}]},
|
||||
{"input": [{"type": "compaction", "encrypted_content": "opaque"}]},
|
||||
{"input": [{"type": "item_reference", "id": "item_parent"}]},
|
||||
{"input": [{"role": "assistant", "content": "visible", "encrypted_content": "opaque"}]},
|
||||
{"input": [{"role": "user", "content": [{"type": "encrypted_content", "encrypted_content": "opaque"}]}]},
|
||||
])
|
||||
@pytest.mark.parametrize("arm_first", [False, True])
|
||||
async def test_client_owned_history_bypasses_compaction(
|
||||
wire: tuple[respx.Route, respx.Route], owned: Mapping[str, object], arm_first: bool,
|
||||
) -> None:
|
||||
router: Final = make_router()
|
||||
deployment: Final = router.get_deployment(model_id="pinned-answer")
|
||||
assert deployment is not None
|
||||
state: Final = CompactionState()
|
||||
request: Final = {**history("responses"), **owned, "model": "small", "max_tokens": 64, "_context_compaction_state": state}
|
||||
original: Final = deepcopy({key: value for key, value in request.items() if key != "_context_compaction_state"})
|
||||
before_defaults: Final = {"_context_compaction_state": state} if arm_first else request
|
||||
arm_compaction(before_defaults, ContextCompactionConfig(), ("large",))
|
||||
result: Final = await compact_to_fit(router, deployment.model_dump(), request, "responses")
|
||||
assert result is request
|
||||
assert {key: value for key, value in result.items() if key != "_context_compaction_state"} == original
|
||||
assert all(route.call_count == 0 for route in wire)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["conflicting_defaults", "small_window", "capability_false", "capability_missing"])
|
||||
async def test_automatic_compactor_skips_conflicts_and_requires_capacity_and_capability(
|
||||
wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, case: str,
|
||||
) -> None:
|
||||
if case.startswith("capability"):
|
||||
metadata: Final = {key: value for key, value in litellm.model_cost["summary-fixture"].items()
|
||||
if key != "supports_anthropic_compaction"}
|
||||
monkeypatch.setitem(litellm.model_cost, "summary-fixture", {
|
||||
**metadata, **({"supports_anthropic_compaction": False} if case == "capability_false" else {}),
|
||||
})
|
||||
conflict: Final = case == "conflicting_defaults"
|
||||
settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "backup", "REASONING": "backup"}} if conflict else {}
|
||||
router: Final = make_router(settings=settings, conflict=conflict, compactor_window=512 if case == "small_window" else 32000)
|
||||
if conflict:
|
||||
await invoke(router, "chat", history("chat"))
|
||||
compactor, answer = wire
|
||||
assert compactor.call_count == answer.call_count == 1
|
||||
assert compactor.calls[0].request.headers["x-api-key"] == "backup-test"
|
||||
assert "stop_sequences" not in json.loads(compactor.calls[0].request.content)
|
||||
else:
|
||||
with pytest.raises(litellm.BadRequestError, match="No configured compactor"):
|
||||
await invoke(router, "chat", history("chat"))
|
||||
assert all(route.call_count == 0 for route in wire)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("unknown_output", [False, True])
|
||||
@pytest.mark.parametrize("overflow", [False, True])
|
||||
async def test_unusable_output_budget_still_enforces_known_input_window(
|
||||
wire: tuple[respx.Route, respx.Route], unknown_output: bool, overflow: bool,
|
||||
) -> None:
|
||||
payload: Final = history("chat")
|
||||
counted: Final = make_router(output=None)._count_pre_call_check_tokens(payload["messages"], None, payload)
|
||||
window: Final = 512 if overflow else counted + 64
|
||||
output: Final = None if unknown_output else window
|
||||
router: Final = make_router(window, output=output)
|
||||
deployment: Final = router.get_deployment(model_id="pinned-answer")
|
||||
assert deployment is not None
|
||||
state: Final = CompactionState(config=ContextCompactionConfig(), candidates=("large",))
|
||||
request: Final = {**payload, "model": "small", "max_tokens": output, "_context_compaction_state": state}
|
||||
if overflow:
|
||||
with pytest.raises(litellm.BadRequestError, match="known input window"):
|
||||
await compact_to_fit(router, deployment.model_dump(), request, "chat")
|
||||
else:
|
||||
assert await compact_to_fit(router, deployment.model_dump(), request, "chat") is request
|
||||
assert all(route.call_count == 0 for route in wire)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["success", "timeout", "cancel"])
|
||||
async def test_retry_reuses_summary_or_terminal_cancellation(outcome: Literal["success", "timeout", "cancel"]) -> None:
|
||||
router: Final = make_router()
|
||||
deployment: Final = router.get_deployment(model_id="pinned-answer")
|
||||
assert deployment is not None
|
||||
state: Final = CompactionState(config=ContextCompactionConfig(model="large", max_tokens=512, timeout_seconds=0.02))
|
||||
request: Final = {**history("messages"), "model": "small", "max_tokens": 64, "_context_compaction_state": state}
|
||||
calls: Final = asyncio.Queue[None]()
|
||||
started: Final = asyncio.Event()
|
||||
stopped: Final = asyncio.Event()
|
||||
|
||||
async def execute(protocol: native.Protocol, payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
calls.put_nowait(None)
|
||||
started.set()
|
||||
try:
|
||||
return native_reply().json() if outcome == "success" else await asyncio.Future[Mapping[str, object]]()
|
||||
finally:
|
||||
stopped.set()
|
||||
|
||||
token: Final = compaction_executor.set(execute)
|
||||
try:
|
||||
first: Final = asyncio.create_task(compact_to_fit(router, deployment.model_dump(), request, "messages"))
|
||||
await asyncio.wait_for(started.wait(), timeout=2)
|
||||
if outcome == "cancel":
|
||||
first.cancel()
|
||||
if outcome == "success":
|
||||
assert await first == await compact_to_fit(router, deployment.model_dump(), request, "messages")
|
||||
changed: Final = {**request, "messages": [{"role": "user", "content": "new history"}, *request["messages"]]}
|
||||
with pytest.raises(litellm.BadRequestError, match="History changed"):
|
||||
await compact_to_fit(router, deployment.model_dump(), changed, "messages")
|
||||
else:
|
||||
error: Final = asyncio.CancelledError if outcome == "cancel" else asyncio.TimeoutError
|
||||
with pytest.raises(error):
|
||||
await first
|
||||
with pytest.raises(error):
|
||||
await compact_to_fit(router, deployment.model_dump(), request, "messages")
|
||||
assert calls.qsize() == 1 and stopped.is_set()
|
||||
finally:
|
||||
compaction_executor.reset(token)
|
||||
|
|
@ -15035,7 +15035,9 @@ class TestHealthFallbackDispatch:
|
|||
) -> None:
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
||||
router: Final = self._router(config={"tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}})
|
||||
router: Final = self._router(
|
||||
config={"context_compaction": False, "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}}
|
||||
)
|
||||
router.add_deployment(
|
||||
Deployment(
|
||||
model_name="large",
|
||||
|
|
@ -15112,7 +15114,9 @@ class TestHealthFallbackDispatch:
|
|||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("default_fits", [True, False])
|
||||
async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None:
|
||||
router: Final = self._router(config={"modality_routing": True, "tiers": {"SIMPLE": "primary"}})
|
||||
router: Final = self._router(
|
||||
config={"context_compaction": False, "modality_routing": True, "tiers": {"SIMPLE": "primary"}}
|
||||
)
|
||||
for deployment in router.model_list:
|
||||
deployment["model_info"]["supports_vision"] = deployment["model_name"] == "fallback"
|
||||
deployment["model_info"]["max_input_tokens"] = 10000 if default_fits else 10
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ class TestAnthropicBetaHeadersFiltering:
|
|||
filtered = filter_and_transform_beta_headers(
|
||||
beta_headers=all_headers, provider=provider
|
||||
)
|
||||
assert ("compact-2026-09-04" in filtered) is (provider == "anthropic")
|
||||
|
||||
for header in unsupported_headers:
|
||||
assert (
|
||||
|
|
|
|||
|
|
@ -178,6 +178,14 @@ def test_potential_model_names_keeps_provider_prefixed_candidate():
|
|||
assert bare["provider_prefixed_model_name"] == bare["combined_model_name"] == "perplexity/glm-5.2"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capability", [True, False, None])
|
||||
def test_get_model_info_anthropic_compaction(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, capability: bool | None
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost["claude-sonnet-5"], "supports_anthropic_compaction", capability)
|
||||
assert litellm.get_model_info("claude-sonnet-5")["supports_anthropic_compaction"] is capability
|
||||
|
||||
|
||||
def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local_model_cost_map):
|
||||
info = litellm.get_model_info(model="ft:gpt-4o-2024-08-06:my-org::abc123", custom_llm_provider="openai")
|
||||
assert info["key"] == "ft:gpt-4o-2024-08-06"
|
||||
|
|
@ -855,6 +863,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"source": {"type": "string"},
|
||||
"comment": {"type": "string"},
|
||||
"supports_assistant_prefill": {"type": "boolean"},
|
||||
"supports_anthropic_compaction": {"type": "boolean"},
|
||||
"supports_audio_input": {"type": "boolean"},
|
||||
"supports_audio_output": {"type": "boolean"},
|
||||
"gemini_native_audio": {"type": "boolean"},
|
||||
|
|
|
|||
25
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
25
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -27419,6 +27419,26 @@ export interface components {
|
|||
*/
|
||||
pattern_type: "prebuilt" | "regex";
|
||||
};
|
||||
/** ContextCompactionConfig */
|
||||
ContextCompactionConfig: {
|
||||
/**
|
||||
* Max Tokens
|
||||
* @default 4096
|
||||
*/
|
||||
max_tokens: number;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
/**
|
||||
* Timeout Seconds
|
||||
* @default 120
|
||||
*/
|
||||
timeout_seconds: number;
|
||||
/**
|
||||
* Trigger Ratio
|
||||
* @default 0.9
|
||||
*/
|
||||
trigger_ratio: number;
|
||||
};
|
||||
/**
|
||||
* CoordinationRedisNode
|
||||
* @description A single startup node of a cluster-mode Redis used for proxy coordination.
|
||||
|
|
@ -36604,6 +36624,11 @@ export interface components {
|
|||
* @description Keywords indicating code-related content
|
||||
*/
|
||||
code_keywords?: string[] | null;
|
||||
/**
|
||||
* Context Compaction
|
||||
* @description Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.
|
||||
*/
|
||||
context_compaction?: components["schemas"]["ContextCompactionConfig"] | false;
|
||||
/**
|
||||
* Context Window Escalation Buffer
|
||||
* @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue