mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(router): compact oversized auto-router requests with native providers
This commit is contained in:
parent
3c34b92594
commit
1276df0b03
29 changed files with 2945 additions and 68 deletions
|
|
@ -7,6 +7,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
|
|||
151
litellm/litellm_core_utils/prompt_templates/compaction.py
Normal file
151
litellm/litellm_core_utils/prompt_templates/compaction.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import StrictStr, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
NativeProtocol: TypeAlias = Literal["responses", "messages"]
|
||||
_Items: TypeAlias = tuple[Mapping[str, object], ...]
|
||||
_ITEMS: Final = TypeAdapter(_Items)
|
||||
_WIRE_ITEMS: Final = TypeAdapter(list[Mapping[str, object]])
|
||||
_HEADERS: Final = TypeAdapter(Mapping[StrictStr, StrictStr])
|
||||
_COMPACTION_BETA: Final = "compact-2026-09-04"
|
||||
|
||||
|
||||
class _CompactionOperation(TypedDict):
|
||||
type: ReadOnly[Literal["summarize"]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompactionHistoryError:
|
||||
message: str
|
||||
kind: Literal["compaction_history_error"] = "compaction_history_error"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeHistory:
|
||||
field: str
|
||||
prefix: _Items
|
||||
tail: _Items
|
||||
|
||||
|
||||
def _copy_items(items: object) -> _Items:
|
||||
return tuple(MappingProxyType(item) for item in deepcopy(_ITEMS.validate_python(items)))
|
||||
|
||||
|
||||
def _native_items(value: object) -> _Items | CompactionHistoryError:
|
||||
try:
|
||||
return _copy_items(value)
|
||||
except ValidationError:
|
||||
return CompactionHistoryError("Native history requires an array of objects")
|
||||
|
||||
|
||||
def _is_user_request(item: Mapping[str, object], protocol: NativeProtocol) -> bool:
|
||||
if item.get("role") != "user":
|
||||
return False
|
||||
content: Final = item.get("content")
|
||||
if protocol == "responses" or isinstance(content, str):
|
||||
return True
|
||||
blocks: Final = _native_items(content)
|
||||
return isinstance(blocks, tuple) and any(block.get("type") != "tool_result" for block in blocks)
|
||||
|
||||
|
||||
def split_native_history(
|
||||
payload: Mapping[str, object], protocol: NativeProtocol
|
||||
) -> NativeHistory | CompactionHistoryError:
|
||||
if payload.get("previous_response_id") is not None or payload.get("conversation") is not None:
|
||||
return CompactionHistoryError("Server-side conversation history is unavailable for native compaction")
|
||||
field: Final = "input" if protocol == "responses" else "messages"
|
||||
items: Final = _native_items(payload.get(field))
|
||||
if isinstance(items, CompactionHistoryError):
|
||||
return items
|
||||
start: Final = next(
|
||||
(index for index in range(len(items) - 1, -1, -1) if _is_user_request(items[index], protocol)),
|
||||
-1,
|
||||
)
|
||||
if start < 0:
|
||||
return CompactionHistoryError("Native history has no actual user request")
|
||||
if start == 0:
|
||||
return CompactionHistoryError("Native history has no compactable prefix")
|
||||
return NativeHistory(field=field, prefix=items[:start], tail=items[start:])
|
||||
|
||||
|
||||
def compaction_headers(existing: object) -> Mapping[str, str]:
|
||||
headers: Final = _HEADERS.validate_python(MappingProxyType({}) if existing is None else existing)
|
||||
betas: Final = tuple(
|
||||
beta.strip()
|
||||
for name, value in headers.items()
|
||||
if name.lower() == "anthropic-beta"
|
||||
for beta in value.split(",")
|
||||
if beta.strip()
|
||||
)
|
||||
preserved: Final = MappingProxyType(
|
||||
{name: value for name, value in headers.items() if name.lower() != "anthropic-beta"}
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**preserved,
|
||||
"anthropic-beta": ",".join(dict.fromkeys((*betas, _COMPACTION_BETA))),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def native_compaction_payload(
|
||||
history: NativeHistory, payload: Mapping[str, object], protocol: NativeProtocol
|
||||
) -> Mapping[str, object]:
|
||||
prefix: Final = deepcopy(_WIRE_ITEMS.validate_python(history.prefix))
|
||||
parameters: Final = ("instructions",) if protocol == "responses" else ("system", "tools")
|
||||
context: Final = MappingProxyType({key: deepcopy(payload[key]) for key in parameters if key in payload})
|
||||
if protocol == "responses":
|
||||
return MappingProxyType({"input": prefix, **context})
|
||||
operation: Final[_CompactionOperation] = {"type": "summarize"}
|
||||
return MappingProxyType(
|
||||
{
|
||||
"messages": prefix,
|
||||
**context,
|
||||
"compaction": operation,
|
||||
"max_tokens": 4096,
|
||||
"extra_headers": _HEADERS.validate_python(compaction_headers(payload.get("extra_headers"))),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_text(item: Mapping[str, object], field: str) -> bool:
|
||||
value: Final = item.get(field)
|
||||
return isinstance(value, str) and bool(value.strip())
|
||||
|
||||
|
||||
def native_compaction_result(
|
||||
response: Mapping[str, object], history: NativeHistory, protocol: NativeProtocol
|
||||
) -> _Items | CompactionHistoryError:
|
||||
if protocol == "responses":
|
||||
output: Final = _native_items(response.get("output"))
|
||||
if isinstance(output, CompactionHistoryError):
|
||||
return output
|
||||
compacted: Final = tuple(item for item in output if item.get("type") == "compaction")
|
||||
if not compacted or any(not _has_text(item, "encrypted_content") for item in compacted):
|
||||
return CompactionHistoryError("Native Responses compaction requires encrypted compaction output")
|
||||
canonical: Final = tuple(
|
||||
MappingProxyType({key: value for key, value in item.items() if key != "created_by" or value is not None})
|
||||
if item.get("type") == "compaction"
|
||||
else item
|
||||
for item in output
|
||||
)
|
||||
return _ITEMS.validate_python((*canonical, *_copy_items(history.tail)))
|
||||
blocks: Final = _native_items(response.get("content"))
|
||||
if response.get("stop_reason") != "compaction" or isinstance(blocks, CompactionHistoryError) or len(blocks) != 1:
|
||||
return CompactionHistoryError("Native Messages compaction requires one signed compaction block")
|
||||
block: Final = blocks[0]
|
||||
if block.get("type") != "compaction" or not _has_text(block, "content") or not _has_text(block, "signature"):
|
||||
return CompactionHistoryError("Native Messages compaction requires content and signature")
|
||||
return _ITEMS.validate_python(
|
||||
(
|
||||
MappingProxyType({"role": "assistant", "content": _WIRE_ITEMS.validate_python(blocks)}),
|
||||
*_copy_items(history.tail),
|
||||
)
|
||||
)
|
||||
|
|
@ -4,18 +4,30 @@ Anthropic CountTokens API handler.
|
|||
Uses httpx for HTTP requests instead of the Anthropic SDK.
|
||||
"""
|
||||
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import Field, StrictStr, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.litellm_core_utils.prompt_templates.compaction import compaction_headers
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
||||
_NATIVE_COUNT_FIELDS: Final = frozenset({"messages", "system", "tools", "tool_choice", "thinking"})
|
||||
_NATIVE_HEADERS: Final = TypeAdapter(dict[StrictStr, StrictStr])
|
||||
_NATIVE_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
_NATIVE_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
|
||||
_NATIVE_COUNT: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field(strict=True, ge=0)])
|
||||
|
||||
|
||||
class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
||||
"""
|
||||
|
|
@ -24,6 +36,76 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
Uses httpx for HTTP requests, following the same pattern as BedrockCountTokensHandler.
|
||||
"""
|
||||
|
||||
async def count_native_tokens(
|
||||
self,
|
||||
model: str,
|
||||
payload: Mapping[str, object],
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
headers: Mapping[str, str],
|
||||
timeout: float,
|
||||
) -> int:
|
||||
handler: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
try:
|
||||
body: Final = _NATIVE_OBJECT.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
**MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in payload.items()
|
||||
if key in _NATIVE_COUNT_FIELDS and value is not None
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
config: Final = AnthropicMessagesConfig()
|
||||
messages_url: Final = httpx.URL(
|
||||
config.get_complete_url(
|
||||
api_base=api_base or litellm.api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
optional_params=_NATIVE_OBJECT.validate_python(MappingProxyType({})),
|
||||
litellm_params=_NATIVE_OBJECT.validate_python(MappingProxyType({})),
|
||||
)
|
||||
)
|
||||
compact_headers: Final = compaction_headers(headers)
|
||||
beta_headers: Final = _NATIVE_HEADERS.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
**compact_headers,
|
||||
"anthropic-beta": ",".join(
|
||||
dict.fromkeys(
|
||||
(*compact_headers["anthropic-beta"].split(","), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION)
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
anthropic_headers: Final = _NATIVE_HEADERS.validate_python(
|
||||
config.validate_anthropic_messages_environment(
|
||||
headers=beta_headers,
|
||||
model=model,
|
||||
messages=_NATIVE_MESSAGES.validate_python(payload.get("messages")),
|
||||
optional_params=body,
|
||||
litellm_params=_NATIVE_OBJECT.validate_python(MappingProxyType({})),
|
||||
api_key=api_key or litellm.anthropic_key or litellm.api_key,
|
||||
api_base=str(messages_url),
|
||||
)[0]
|
||||
)
|
||||
response: Final = await handler.client.post(
|
||||
str(messages_url.copy_with(path=f"{messages_url.path.rstrip('/')}/count_tokens")),
|
||||
headers=anthropic_headers,
|
||||
json=body,
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _NATIVE_COUNT.validate_python(_NATIVE_OBJECT.validate_json(response.content).get("input_tokens"))
|
||||
except (httpx.HTTPError, ValueError):
|
||||
raise ValueError("Provider-native token counting failed") from None
|
||||
|
||||
async def handle_count_tokens_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
"compaction",
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
|
|||
|
|
@ -5,9 +5,13 @@ Uses httpx for HTTP requests to OpenAI's /v1/responses/input_tokens endpoint.
|
|||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, Final
|
||||
|
||||
import httpx
|
||||
from openai import APIError, AsyncOpenAI
|
||||
from pydantic import Field, StrictStr, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -16,6 +20,16 @@ from litellm.llms.openai.common_utils import OpenAIError
|
|||
from litellm.llms.openai.responses.count_tokens.transformation import (
|
||||
OpenAICountTokensConfig,
|
||||
)
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_NATIVE_COUNT_FIELDS: Final = frozenset(
|
||||
{"input", "instructions", "tools", "tool_choice", "parallel_tool_calls", "reasoning", "text"}
|
||||
)
|
||||
_NATIVE_HEADERS: Final = TypeAdapter(dict[StrictStr, StrictStr])
|
||||
_NATIVE_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
_NATIVE_COUNT: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field(strict=True, ge=0)])
|
||||
|
||||
|
||||
class OpenAICountTokensHandler(OpenAICountTokensConfig):
|
||||
|
|
@ -23,6 +37,64 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
|
|||
Handler for OpenAI Responses API token counting requests.
|
||||
"""
|
||||
|
||||
async def count_native_tokens(
|
||||
self,
|
||||
model: str,
|
||||
payload: Mapping[str, object],
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
headers: Mapping[str, str],
|
||||
timeout: float,
|
||||
) -> int:
|
||||
handler: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI)
|
||||
try:
|
||||
body: Final = _NATIVE_OBJECT.validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
**MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in payload.items()
|
||||
if key in _NATIVE_COUNT_FIELDS and value is not None
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
config: Final = OpenAIResponsesAPIConfig()
|
||||
openai_base: Final = config.get_complete_url(
|
||||
api_base=api_base, litellm_params=_NATIVE_OBJECT.validate_python(MappingProxyType({}))
|
||||
).removesuffix("/responses")
|
||||
openai_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
|
||||
environment_headers: Final = _NATIVE_HEADERS.validate_python(
|
||||
config.validate_environment(
|
||||
headers=_NATIVE_HEADERS.validate_python(headers),
|
||||
model=model,
|
||||
litellm_params=GenericLiteLLMParams(api_key=openai_key, api_base=openai_base),
|
||||
)
|
||||
)
|
||||
client: Final = AsyncOpenAI(
|
||||
api_key=openai_key or "",
|
||||
base_url=openai_base,
|
||||
http_client=handler.client,
|
||||
max_retries=0,
|
||||
)
|
||||
sdk_header_names: Final = MappingProxyType({name.lower(): name for name in client.default_headers})
|
||||
openai_headers: Final = MappingProxyType(
|
||||
{
|
||||
sdk_header_names.get(name.lower(), name): value
|
||||
for source in (environment_headers, headers)
|
||||
for name, value in source.items()
|
||||
}
|
||||
)
|
||||
result: Final = await client.responses.input_tokens.count(
|
||||
model=model, extra_body=body, extra_headers=openai_headers, timeout=timeout
|
||||
)
|
||||
return _NATIVE_COUNT.validate_python(result.input_tokens)
|
||||
except (APIError, httpx.HTTPError, ValueError):
|
||||
raise ValueError("Provider-native token counting failed") from None
|
||||
|
||||
async def handle_count_tokens_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -898,11 +898,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
self,
|
||||
model: str,
|
||||
input: str | ResponseInputParam,
|
||||
response_api_optional_request_params: dict,
|
||||
response_api_optional_request_params: Mapping[str, object],
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> tuple[str, dict]:
|
||||
headers: Mapping[str, object],
|
||||
) -> tuple[str, dict[str, object]]:
|
||||
"""
|
||||
Transform the compact response API request into a URL and data
|
||||
|
||||
|
|
@ -914,16 +914,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
compact_path: Final = parsed_url.path.rstrip("/") + "/compact"
|
||||
url: Final = str(parsed_url.copy_with(path=compact_path))
|
||||
|
||||
replay_safe_input, sanitized_tools = self._prepared_input_and_tools(
|
||||
replay_safe_input, _ = self._prepared_input_and_tools(
|
||||
model=model,
|
||||
input=input,
|
||||
tools=response_api_optional_request_params.get("tools"),
|
||||
tools=None,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
if sanitized_tools is not None:
|
||||
response_api_optional_request_params["tools"] = sanitized_tools
|
||||
data: Final = dict(
|
||||
ResponsesAPIRequestParams(model=model, input=replay_safe_input, **response_api_optional_request_params)
|
||||
compact_fields: Final = frozenset(
|
||||
{"instructions", "previous_response_id", "prompt_cache_key", "prompt_cache_retention"}
|
||||
)
|
||||
data: Final = TypeAdapter(dict[str, object]).validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"input": replay_safe_input,
|
||||
**MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in response_api_optional_request_params.items()
|
||||
if key in compact_fields
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return url, data
|
||||
|
|
|
|||
|
|
@ -75,6 +75,11 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
get_logging_caching_headers,
|
||||
get_remaining_tokens_and_requests_from_request_data,
|
||||
)
|
||||
from litellm.proxy.common_utils.context_compaction import (
|
||||
mark_summary_provider_completed,
|
||||
proxy_summary_executor_scope,
|
||||
summary_request_is_private,
|
||||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
get_client_requested_model,
|
||||
get_tags_from_request_body,
|
||||
|
|
@ -2336,7 +2341,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
def _debug_log_request_payload(self) -> None:
|
||||
"""Log request payload at DEBUG level, truncating if too large."""
|
||||
if not verbose_proxy_logger.isEnabledFor(logging.DEBUG):
|
||||
if summary_request_is_private() or not verbose_proxy_logger.isEnabledFor(logging.DEBUG):
|
||||
return
|
||||
_payload_str: Final = json.dumps(self.data, default=str)
|
||||
if len(_payload_str) > MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG:
|
||||
|
|
@ -2517,20 +2522,24 @@ class ProxyBaseLLMRequestProcessing:
|
|||
user_model=user_model,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
llm_call_task: Final = asyncio.create_task(llm_call)
|
||||
tasks.append(llm_call_task)
|
||||
with proxy_summary_executor_scope(
|
||||
request, self.data, getattr(proxy_logging_obj, "max_parallel_request_limiter", None)
|
||||
):
|
||||
llm_call_task: Final = asyncio.create_task(llm_call)
|
||||
tasks.append(llm_call_task)
|
||||
|
||||
llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call
|
||||
llm_responses: Final = asyncio.gather(*tasks)
|
||||
|
||||
try:
|
||||
if general_settings.get("cancel_on_disconnect", False):
|
||||
responses = await _await_llm_call_cancelling_on_disconnect(request, llm_responses)
|
||||
else:
|
||||
responses = await llm_responses
|
||||
finally:
|
||||
await _cancel_pending_gather_tasks(tasks)
|
||||
try:
|
||||
if general_settings.get("cancel_on_disconnect", False):
|
||||
responses = await _await_llm_call_cancelling_on_disconnect(request, llm_responses)
|
||||
else:
|
||||
responses = await llm_responses
|
||||
finally:
|
||||
await _cancel_pending_gather_tasks(tasks)
|
||||
|
||||
response = responses[1]
|
||||
mark_summary_provider_completed()
|
||||
|
||||
_exception_raised = False
|
||||
try:
|
||||
|
|
|
|||
278
litellm/proxy/common_utils/context_compaction.py
Normal file
278
litellm/proxy/common_utils/context_compaction.py
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
import httpx
|
||||
from fastapi import Request
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
effective_turn_off_message_logging,
|
||||
parent_session_kwargs,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
RequestRateLimiterStash,
|
||||
finish_summary_cleanup,
|
||||
get_request_stash,
|
||||
isolated_summary_request_stash,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
release_budget_reservation_on_cancel, # pyright: ignore[reportUnknownVariableType] # legacy reservation dict is untyped
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.context_compaction import NativeRequest
|
||||
from litellm.types.utils import CONTEXT_COMPACTION_CALL_ORIGIN
|
||||
|
||||
_ASGI_APP: Final[TypeAdapter[ASGIApp]] = TypeAdapter(ASGIApp)
|
||||
_STRING: Final = TypeAdapter(str)
|
||||
_JSON_BODY: Final = TypeAdapter(Mapping[str, JsonValue])
|
||||
_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
_METADATA: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_BODY_FIELDS: Final = frozenset(
|
||||
{"input", "instructions", "messages", "max_tokens", "system", "tools", "compaction"}
|
||||
)
|
||||
_OMITTED_HEADERS: Final = frozenset(
|
||||
{
|
||||
"content-length",
|
||||
"content-type",
|
||||
"transfer-encoding",
|
||||
"x-litellm-call-id",
|
||||
"x-app",
|
||||
"x-claude-code-agent-id",
|
||||
"x-litellm-model",
|
||||
"x-litellm-model-id",
|
||||
"litellm-disable-message-redaction",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SummaryRequestLimiter(Protocol):
|
||||
async def release_summary_request_capacity(
|
||||
self, auth: UserAPIKeyAuth, request_data: Mapping[str, object], provider_completed: bool
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _SummaryRequest:
|
||||
call_id: str
|
||||
redact: bool | None
|
||||
auth: UserAPIKeyAuth | None = None
|
||||
data: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
|
||||
provider_completed: bool = False
|
||||
closed: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
def bind(self, auth: UserAPIKeyAuth, data: Mapping[str, object]) -> None:
|
||||
self.auth = auth
|
||||
self.data = data
|
||||
if (
|
||||
self.redact is True
|
||||
or effective_turn_off_message_logging(data) is True
|
||||
or litellm.turn_off_message_logging is True
|
||||
):
|
||||
self.redact = True
|
||||
|
||||
|
||||
_summary_request: Final[ContextVar[_SummaryRequest | None]] = ContextVar(
|
||||
"litellm_context_compaction_request", default=None
|
||||
)
|
||||
|
||||
|
||||
def summary_request_updates(
|
||||
request: Request,
|
||||
auth: UserAPIKeyAuth,
|
||||
data: Mapping[str, object],
|
||||
metadata_key: str,
|
||||
) -> Mapping[str, object]:
|
||||
summary: Final = _summary_request.get()
|
||||
if summary is None or summary.closed.is_set() or request.headers.get("x-litellm-call-id") != summary.call_id:
|
||||
return MappingProxyType({})
|
||||
summary.bind(auth, data)
|
||||
metadata: Final = _METADATA.validate_python(data.get(metadata_key) or MappingProxyType({}))
|
||||
stamped_metadata: Final = _METADATA.validate_python(
|
||||
MappingProxyType({**metadata, INTERNAL_CALL_ORIGIN_METADATA_KEY: CONTEXT_COMPACTION_CALL_ORIGIN})
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
name: value
|
||||
for name, value in (
|
||||
(metadata_key, stamped_metadata),
|
||||
("num_retries", 0),
|
||||
("max_retries", 0),
|
||||
("disable_fallbacks", True),
|
||||
("turn_off_message_logging", True if summary.redact is True else None),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def mark_summary_provider_completed() -> None:
|
||||
summary: Final = _summary_request.get()
|
||||
if summary is not None and not summary.closed.is_set():
|
||||
summary.provider_completed = True
|
||||
|
||||
|
||||
def summary_request_is_private() -> bool:
|
||||
summary: Final = _summary_request.get()
|
||||
return summary is not None and not summary.closed.is_set() and summary.redact is True
|
||||
|
||||
|
||||
async def _settle_cancelled_summary(
|
||||
summary: _SummaryRequest,
|
||||
limiter: SummaryRequestLimiter | None,
|
||||
) -> None:
|
||||
auth: Final = summary.auth
|
||||
if auth is None:
|
||||
return
|
||||
if not summary.provider_completed:
|
||||
await release_budget_reservation_on_cancel(auth.budget_reservation)
|
||||
if limiter is not None:
|
||||
await limiter.release_summary_request_capacity(auth, summary.data, summary.provider_completed)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProxySummaryExecutor:
|
||||
request: Request
|
||||
request_data: Mapping[str, object]
|
||||
limiter: SummaryRequestLimiter | None
|
||||
parent_stash: RequestRateLimiterStash | None
|
||||
closed: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
call_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
|
||||
async def __call__(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
|
||||
if self.closed.is_set() or _summary_request.get() is not None:
|
||||
raise ValueError("Context compaction executor is closed or nested")
|
||||
return await asyncio.wait_for(self._serialized_execute(model, request, timeout), timeout)
|
||||
|
||||
async def _serialized_execute(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
|
||||
async with self.call_lock:
|
||||
if self.closed.is_set():
|
||||
raise ValueError("Context compaction executor is closed")
|
||||
return await self._execute(model, request, timeout)
|
||||
|
||||
async def _execute(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
|
||||
dynamic_redact: Final = effective_turn_off_message_logging(self.request_data)
|
||||
redact: Final = True if litellm.turn_off_message_logging is True else dynamic_redact
|
||||
summary: Final = _SummaryRequest(call_id=str(uuid.uuid4()), redact=redact)
|
||||
parent_call_id: Final = self.request_data.get("litellm_call_id")
|
||||
scope: Final = _METADATA.validate_python(self.request.scope)
|
||||
root_path: Final = _STRING.validate_python(scope.get("root_path", "")).rstrip("/")
|
||||
endpoint: Final = "/v1/responses/compact" if request.protocol == "responses" else "/v1/messages"
|
||||
url: Final = str(self.request.url.replace(path=f"{root_path}{endpoint}"))
|
||||
native_headers: Final = httpx.Headers(
|
||||
_HEADERS.validate_python(request.payload.get("extra_headers") or MappingProxyType({}))
|
||||
)
|
||||
beta: Final = ",".join(
|
||||
dict.fromkeys(
|
||||
token.strip()
|
||||
for value in (self.request.headers.get("anthropic-beta"), native_headers.get("anthropic-beta"))
|
||||
if value
|
||||
for token in value.split(",")
|
||||
if token.strip()
|
||||
)
|
||||
)
|
||||
headers: Final = httpx.Headers(
|
||||
tuple(
|
||||
(name, value)
|
||||
for name, value in self.request.headers.items()
|
||||
if name.lower() not in _OMITTED_HEADERS and name.lower() != "anthropic-beta"
|
||||
)
|
||||
+ ((("anthropic-beta", beta),) if beta else ())
|
||||
+ (("x-litellm-call-id", summary.call_id), ("content-type", "application/json"))
|
||||
)
|
||||
session: Final = parent_session_kwargs(self.request_data)
|
||||
region: Final = self.request_data.get("allowed_model_region")
|
||||
user: Final = self.request_data.get("user")
|
||||
payload: Final = MappingProxyType(
|
||||
{
|
||||
**MappingProxyType(
|
||||
{name: value for name, value in request.payload.items() if name in _NATIVE_BODY_FIELDS}
|
||||
),
|
||||
"allowed_model_region": region if isinstance(region, str) else None,
|
||||
"user": user if isinstance(user, str) else None,
|
||||
"litellm_session_id": session.get("litellm_session_id"),
|
||||
"litellm_trace_id": session.get("litellm_trace_id"),
|
||||
"model": model,
|
||||
"timeout": timeout,
|
||||
"stream": False if request.protocol == "messages" else None,
|
||||
"num_retries": 0,
|
||||
"max_retries": 0,
|
||||
"disable_fallbacks": True,
|
||||
"metadata": MappingProxyType({"tags": tuple(get_tags_from_request_body(self.request_data))}),
|
||||
}
|
||||
)
|
||||
body: Final = json.dumps(
|
||||
MappingProxyType({name: value for name, value in payload.items() if value is not None}),
|
||||
default=dict,
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
address: Final = self.request.client
|
||||
transport: Final = httpx.ASGITransport(
|
||||
app=_ASGI_APP.validate_python(scope["app"]),
|
||||
root_path=root_path,
|
||||
client=(address.host, address.port) if address is not None else ("127.0.0.1", 0),
|
||||
raise_app_exceptions=False,
|
||||
)
|
||||
async with isolated_summary_request_stash(
|
||||
self.parent_stash, parent_call_id if isinstance(parent_call_id, str) else None, summary.call_id
|
||||
):
|
||||
token: Final = _summary_request.set(summary)
|
||||
try:
|
||||
async with transport:
|
||||
response: Final = await transport.handle_async_request(
|
||||
httpx.Request("POST", url, headers=headers, content=body)
|
||||
)
|
||||
try:
|
||||
if not response.is_success:
|
||||
raise litellm.APIError(
|
||||
status_code=response.status_code,
|
||||
message="Context compaction request failed",
|
||||
model=model,
|
||||
llm_provider="litellm",
|
||||
)
|
||||
return _JSON_BODY.validate_json(await response.aread())
|
||||
finally:
|
||||
await response.aclose()
|
||||
except (asyncio.CancelledError, TimeoutError):
|
||||
await finish_summary_cleanup(_settle_cancelled_summary(summary, self.limiter))
|
||||
raise
|
||||
finally:
|
||||
summary.closed.set()
|
||||
_summary_request.reset(token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def proxy_summary_executor_scope(
|
||||
request: Request,
|
||||
request_data: Mapping[str, object],
|
||||
limiter: object,
|
||||
) -> Generator[None]:
|
||||
from litellm.router_strategy.complexity_router.context_compaction import use_summary_executor
|
||||
|
||||
if _summary_request.get() is not None:
|
||||
yield
|
||||
return
|
||||
executor: Final = ProxySummaryExecutor(
|
||||
request=request,
|
||||
request_data=request_data,
|
||||
limiter=limiter if isinstance(limiter, SummaryRequestLimiter) else None,
|
||||
parent_stash=get_request_stash(),
|
||||
)
|
||||
try:
|
||||
with use_summary_executor(executor):
|
||||
yield
|
||||
finally:
|
||||
executor.closed.set()
|
||||
|
|
@ -10,7 +10,7 @@ import logging
|
|||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence, Set
|
||||
from contextlib import asynccontextmanager
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
|
@ -66,6 +66,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
|
||||
from litellm.types.utils import (
|
||||
CONTEXT_COMPACTION_CALL_ORIGIN,
|
||||
CallTypes,
|
||||
EmbeddingResponse,
|
||||
ModelResponse,
|
||||
|
|
@ -428,8 +429,8 @@ class ParallelRequestGauge(TypedDict):
|
|||
|
||||
|
||||
class ParallelSlotAcquisition(TypedDict):
|
||||
slot_id: str
|
||||
counter_keys: list[str]
|
||||
slot_id: ReadOnly[str]
|
||||
counter_keys: ReadOnly[Sequence[str]]
|
||||
|
||||
|
||||
class RateLimitStatus(TypedDict):
|
||||
|
|
@ -559,6 +560,76 @@ def get_request_stash() -> RequestRateLimiterStash | None:
|
|||
return _request_stash.get()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BorrowedParallelSlots:
|
||||
stash: RequestRateLimiterStash
|
||||
call_id: str
|
||||
counter_keys: frozenset[str]
|
||||
consumed: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
_borrowed_parallel_slots: Final[ContextVar[_BorrowedParallelSlots | None]] = ContextVar(
|
||||
"litellm_summary_borrowed_parallel_slots", default=None
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def isolated_summary_request_stash(
|
||||
parent_stash: RequestRateLimiterStash | None,
|
||||
parent_call_id: str | None,
|
||||
child_call_id: str,
|
||||
) -> AsyncGenerator[RequestRateLimiterStash, None]:
|
||||
async with AsyncExitStack() as stack:
|
||||
eligible_parent: Final = (
|
||||
parent_stash
|
||||
if parent_stash is not None
|
||||
and parent_call_id is not None
|
||||
and parent_stash.owner_litellm_call_id == parent_call_id
|
||||
and _borrowed_parallel_slots.get() is None
|
||||
else None
|
||||
)
|
||||
if eligible_parent is not None:
|
||||
await stack.enter_async_context(eligible_parent.parallel_slot_release_lock)
|
||||
acquisition: Final = eligible_parent.parallel_slot if eligible_parent is not None else None
|
||||
stash: Final = RequestRateLimiterStash(owner_litellm_call_id=child_call_id)
|
||||
stash_token: Final = _request_stash.set(stash)
|
||||
borrow_token: Final = _borrowed_parallel_slots.set(
|
||||
_BorrowedParallelSlots(
|
||||
stash=stash,
|
||||
call_id=child_call_id,
|
||||
counter_keys=frozenset(acquisition["counter_keys"]) if acquisition is not None else frozenset(),
|
||||
)
|
||||
)
|
||||
try:
|
||||
yield stash
|
||||
finally:
|
||||
_borrowed_parallel_slots.reset(borrow_token)
|
||||
_request_stash.reset(stash_token)
|
||||
|
||||
|
||||
def _consume_borrowed_parallel_keys(stash: RequestRateLimiterStash) -> frozenset[str]:
|
||||
borrowed: Final = _borrowed_parallel_slots.get()
|
||||
if (
|
||||
borrowed is None
|
||||
or borrowed.stash is not stash
|
||||
or borrowed.call_id != stash.owner_litellm_call_id
|
||||
or borrowed.consumed.is_set()
|
||||
):
|
||||
return frozenset()
|
||||
borrowed.consumed.set()
|
||||
return borrowed.counter_keys
|
||||
|
||||
|
||||
async def finish_summary_cleanup(cleanup: Awaitable[None]) -> None:
|
||||
task: Final = asyncio.ensure_future(cleanup)
|
||||
while not task.done():
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
task.result()
|
||||
|
||||
|
||||
def get_or_create_request_stash() -> RequestRateLimiterStash:
|
||||
stash = _request_stash.get()
|
||||
if stash is None:
|
||||
|
|
@ -1262,6 +1333,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
read_only: bool = False,
|
||||
skip_tpm_check: bool = False,
|
||||
parallel_slot_id: str | None = None,
|
||||
borrowed_parallel_keys: frozenset[str] = frozenset(),
|
||||
) -> RateLimitResponse:
|
||||
"""
|
||||
Check if any of the rate limit descriptors should be rate limited.
|
||||
|
|
@ -1297,6 +1369,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
keys_to_fetch, key_metadata, gauges = self._collect_windowed_keys_and_gauges(
|
||||
descriptors=descriptors,
|
||||
skip_tpm_check=skip_tpm_check,
|
||||
borrowed_parallel_keys=borrowed_parallel_keys,
|
||||
)
|
||||
|
||||
windowed_response = RateLimitResponse(overall_code="OK", statuses=[])
|
||||
|
|
@ -1385,6 +1458,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
self,
|
||||
descriptors: Sequence[RateLimitDescriptor],
|
||||
skip_tpm_check: bool,
|
||||
borrowed_parallel_keys: frozenset[str] = frozenset(),
|
||||
) -> tuple[list[str], dict[str, WindowKeyMetadata], list[ParallelRequestGauge]]:
|
||||
"""
|
||||
Split descriptors into the windowed (window_key, counter_key) fetch
|
||||
|
|
@ -1407,7 +1481,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
window_key = f"{{{descriptor_key}:{descriptor_value}}}:window"
|
||||
|
||||
if max_parallel_requests_limit is not None:
|
||||
if (
|
||||
max_parallel_requests_limit is not None
|
||||
and self.create_rate_limit_keys(descriptor_key, descriptor_value, "max_parallel_requests")
|
||||
not in borrowed_parallel_keys
|
||||
):
|
||||
gauges.append(
|
||||
ParallelRequestGauge(
|
||||
counter_key=self.create_rate_limit_keys(
|
||||
|
|
@ -3492,6 +3570,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if response["overall_code"] == "OK":
|
||||
await self._release_parallel_request_slots(acquisition, user_api_key_dict.parent_otel_span)
|
||||
|
||||
async def _admit_request_capacity(
|
||||
self,
|
||||
descriptors: Sequence[RateLimitDescriptor],
|
||||
auth: UserAPIKeyAuth,
|
||||
slot_id: str | None,
|
||||
counter_keys: Sequence[str],
|
||||
borrowed_parallel_keys: frozenset[str],
|
||||
) -> RateLimitResponse:
|
||||
admission: Final = self.should_rate_limit(
|
||||
descriptors=descriptors,
|
||||
parent_otel_span=auth.parent_otel_span,
|
||||
skip_tpm_check=self.tpm_reservation_enabled,
|
||||
parallel_slot_id=slot_id,
|
||||
borrowed_parallel_keys=borrowed_parallel_keys,
|
||||
)
|
||||
if _borrowed_parallel_slots.get() is None:
|
||||
return await admission
|
||||
task: Final = asyncio.create_task(admission)
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
acquisition: Final = ParallelSlotAcquisition(slot_id=slot_id or "", counter_keys=tuple(counter_keys))
|
||||
await finish_summary_cleanup(self._release_request_capacity_when_admitted(task, acquisition, auth))
|
||||
raise
|
||||
|
||||
@asynccontextmanager
|
||||
async def request_capacity(
|
||||
self,
|
||||
|
|
@ -3552,6 +3655,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook")
|
||||
|
||||
stash: Final = claim_request_stash_for_data(data)
|
||||
borrowed_parallel_keys: Final = _consume_borrowed_parallel_keys(stash)
|
||||
|
||||
#########################################################
|
||||
# Check if the call type has a specific rate limiter
|
||||
|
|
@ -3595,6 +3699,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
|
||||
for d in descriptors
|
||||
if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None
|
||||
and self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
|
||||
not in borrowed_parallel_keys
|
||||
]
|
||||
parallel_slot_id: Final = uuid.uuid4().hex if parallel_counter_keys else None
|
||||
|
||||
|
|
@ -3605,11 +3711,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
d for d in descriptors if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
|
||||
)
|
||||
)
|
||||
response: Final = await self.should_rate_limit(
|
||||
response: Final = await self._admit_request_capacity(
|
||||
descriptors=first_pass_descriptors,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
skip_tpm_check=self.tpm_reservation_enabled,
|
||||
parallel_slot_id=parallel_slot_id,
|
||||
auth=user_api_key_dict,
|
||||
slot_id=parallel_slot_id,
|
||||
counter_keys=parallel_counter_keys,
|
||||
borrowed_parallel_keys=borrowed_parallel_keys,
|
||||
)
|
||||
|
||||
if response["overall_code"] == "OVER_LIMIT":
|
||||
|
|
@ -4429,7 +4536,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# 'metadata' and 'litellm_metadata' fields from litellm_params
|
||||
standard_logging_object: Final = kwargs.get("standard_logging_object") or {}
|
||||
request_metadata: Final = get_litellm_metadata_from_kwargs(kwargs)
|
||||
if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY):
|
||||
call_origin: Final = request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)
|
||||
if call_origin and call_origin != CONTEXT_COMPACTION_CALL_ORIGIN:
|
||||
# Internal sub-calls bill spend to the caller but are not the caller's
|
||||
# traffic; charging them here would let background evals eat TPM headroom.
|
||||
return []
|
||||
|
|
@ -4811,9 +4919,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if popped is not None:
|
||||
await self.batch_enqueued_token_store.refund(reservation=popped, litellm_parent_otel_span=span)
|
||||
|
||||
async def release_summary_request_capacity(
|
||||
self, auth: UserAPIKeyAuth, request_data: Mapping[str, object], provider_completed: bool
|
||||
) -> None:
|
||||
await self._release_stashed_parallel_slot(get_request_stash(), auth.parent_otel_span)
|
||||
if not provider_completed:
|
||||
await self.async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=TimeoutError("Context compaction summary cancelled"),
|
||||
user_api_key_dict=auth,
|
||||
)
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
request_data: Mapping[str, object],
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
traceback_str: str | None = None,
|
||||
|
|
|
|||
|
|
@ -2194,6 +2194,10 @@ async def add_litellm_data_to_request(
|
|||
if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True:
|
||||
_strip_client_message_redaction_opt_out(data)
|
||||
|
||||
from litellm.proxy.common_utils.context_compaction import summary_request_is_private, summary_request_updates
|
||||
|
||||
data.update(summary_request_updates(request, user_api_key_dict, data, _metadata_variable_name))
|
||||
|
||||
# Fill in the proxy_server_request body snapshot now that metadata has
|
||||
# been parsed. Consumers (standard_logging_payload, lago,
|
||||
# spend_tracking_utils, streaming_iterator) read `body` to audit the
|
||||
|
|
@ -2491,7 +2495,8 @@ async def add_litellm_data_to_request(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
|
||||
if not summary_request_is_private():
|
||||
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
|
||||
|
||||
# Team/Project credential overrides from model_config
|
||||
# Placed after the debug log to avoid leaking credential secrets in logs
|
||||
|
|
|
|||
|
|
@ -118,6 +118,21 @@ from litellm.llms.openai_like.model_info import (
|
|||
get_openai_compatible_model_info,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
COMPACTION_STATE_KEY,
|
||||
CompactedRequest,
|
||||
CompactionFailure,
|
||||
CompactionState,
|
||||
ModelBudget,
|
||||
NativeRequest,
|
||||
compaction_state,
|
||||
current_native_request,
|
||||
current_summary_executor,
|
||||
defers_context_filter,
|
||||
prepare_compaction,
|
||||
raise_compaction_failure,
|
||||
validate_native_recipient,
|
||||
)
|
||||
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
|
||||
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
|
||||
|
|
@ -459,10 +474,19 @@ def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
|
|||
|
||||
|
||||
def _silent_experiment_kwargs_snapshot(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
snapshot: Final = MappingProxyType({key: value for key, value in kwargs.items() if key != "messages"})
|
||||
metadata: Final = kwargs.get("metadata")
|
||||
if not isinstance(metadata, Mapping):
|
||||
return MappingProxyType({**kwargs})
|
||||
return MappingProxyType({**kwargs, "metadata": dict(metadata)})
|
||||
return snapshot
|
||||
return MappingProxyType({**snapshot, "metadata": TypeAdapter(dict[str, object]).validate_python(metadata)})
|
||||
|
||||
|
||||
def _compaction_deployment_id(deployment: DeploymentTypedDict) -> str:
|
||||
info: Final = TypeAdapter(Mapping[str, object]).validate_python(
|
||||
deployment.get("model_info") or MappingProxyType({})
|
||||
)
|
||||
identifier: Final = info.get("id")
|
||||
return identifier if isinstance(identifier, str) else ""
|
||||
|
||||
|
||||
def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]:
|
||||
|
|
@ -2617,7 +2641,7 @@ class Router:
|
|||
return silent_kwargs
|
||||
|
||||
async def _run_silent_experiment(
|
||||
self, silent_model: str, messages: Sequence[Mapping[str, str]], silent_kwargs: Mapping[str, object]
|
||||
self, silent_model: str, messages: Sequence[Mapping[str, object]], silent_kwargs: Mapping[str, object]
|
||||
) -> None:
|
||||
remaining_kwargs: Final = MappingProxyType(
|
||||
{key: value for key, value in silent_kwargs.items() if key != "stream"}
|
||||
|
|
@ -2709,6 +2733,7 @@ class Router:
|
|||
kwargs["messages"] = messages
|
||||
kwargs["stream"] = stream
|
||||
kwargs["original_function"] = self._acompletion
|
||||
kwargs[COMPACTION_STATE_KEY] = CompactionState() # rebind-ok: share request-local state across retries
|
||||
|
||||
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs)
|
||||
request_priority: Final = kwargs.get("priority") or self.default_priority
|
||||
|
|
@ -3532,7 +3557,9 @@ class Router:
|
|||
wrapper_ref: Final = weakref.ref(wrapped_response)
|
||||
return wrapped_response
|
||||
|
||||
async def _silent_experiment_acompletion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
|
||||
async def _silent_experiment_acompletion(
|
||||
self, silent_model: str, messages: Sequence[Mapping[str, object]], **kwargs
|
||||
):
|
||||
"""
|
||||
Run a silent experiment in the background.
|
||||
"""
|
||||
|
|
@ -3572,12 +3599,30 @@ class Router:
|
|||
|
||||
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
|
||||
start_time: Final = time.time()
|
||||
deployment = await self.async_get_available_deployment(
|
||||
selected_deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
messages=messages,
|
||||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
compacted_request: Final = await self._aprepare_selected_deployment(
|
||||
deployment=cast( # cast-ok: Router registration validates deployments
|
||||
DeploymentTypedDict, selected_deployment
|
||||
),
|
||||
model=model,
|
||||
request_kwargs=kwargs,
|
||||
messages=messages,
|
||||
protocol="chat",
|
||||
)
|
||||
deployment = selected_deployment # rebind-ok: only prepared deployments count as provider attempts
|
||||
kwargs.pop(COMPACTION_STATE_KEY, None)
|
||||
effective_messages: Final = (
|
||||
TypeAdapter(list[dict[str, object]]).validate_python(compacted_request.value)
|
||||
if compacted_request is not None
|
||||
else messages
|
||||
)
|
||||
if compacted_request is not None:
|
||||
kwargs[compacted_request.field] = effective_messages # rebind-ok: rewrite only this dispatch attempt
|
||||
self._drop_unsupported_classifier_reasoning_effort(
|
||||
deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment
|
||||
model=model,
|
||||
|
|
@ -3615,12 +3660,12 @@ class Router:
|
|||
asyncio.create_task(
|
||||
self._silent_experiment_acompletion(
|
||||
silent_model=silent_target,
|
||||
messages=messages, # Use messages instead of *args
|
||||
messages=effective_messages,
|
||||
**_silent_experiment_kwargs_snapshot(kwargs),
|
||||
)
|
||||
)
|
||||
|
||||
kwargs.setdefault("messages", messages)
|
||||
kwargs.setdefault("messages", effective_messages)
|
||||
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
|
||||
kwargs.pop("silent_model", None) # Ensure it's not in kwargs either
|
||||
|
||||
|
|
@ -3634,7 +3679,7 @@ class Router:
|
|||
|
||||
input_kwargs: Final = {
|
||||
**litellm_params,
|
||||
"messages": messages,
|
||||
"messages": effective_messages,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
|
|
@ -5259,6 +5304,10 @@ class Router:
|
|||
kwargs["model"] = model
|
||||
kwargs["original_generic_function"] = original_function
|
||||
kwargs["original_function"] = self._ageneric_api_call_with_fallbacks_helper
|
||||
if original_function in (litellm.aresponses, litellm.anthropic_messages):
|
||||
kwargs[COMPACTION_STATE_KEY] = ( # rebind-ok: preserve trusted state across generic streaming retries
|
||||
compaction_state(kwargs) or CompactionState()
|
||||
)
|
||||
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata")
|
||||
verbose_router_logger.debug(
|
||||
"Inside ageneric_api_call_with_fallbacks() - model: %s; kwargs: %s", model, kwargs
|
||||
|
|
@ -5318,13 +5367,34 @@ class Router:
|
|||
try:
|
||||
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
|
||||
try:
|
||||
deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init above
|
||||
selected_deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
request_kwargs=kwargs,
|
||||
messages=kwargs.get("messages", None),
|
||||
input=kwargs.get("input", None),
|
||||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
)
|
||||
compacted_request: Final = await self._aprepare_selected_deployment(
|
||||
deployment=cast( # cast-ok: Router registration validates deployments
|
||||
DeploymentTypedDict, selected_deployment
|
||||
),
|
||||
model=model,
|
||||
request_kwargs=kwargs,
|
||||
messages=kwargs.get("messages"),
|
||||
input=kwargs.get("input"),
|
||||
protocol="responses" if kwargs.get("input") is not None else "messages",
|
||||
)
|
||||
deployment = selected_deployment # rebind-ok: only prepared deployments count as provider attempts
|
||||
kwargs.pop(COMPACTION_STATE_KEY, None)
|
||||
if compacted_request is not None:
|
||||
prepared_history: Final = TypeAdapter(list[dict[str, object]]).validate_python(
|
||||
compacted_request.value
|
||||
)
|
||||
kwargs[compacted_request.field] = prepared_history # rebind-ok: rewrite only this dispatch attempt
|
||||
if compacted_request.extra_headers:
|
||||
kwargs["extra_headers"] = TypeAdapter(dict[str, str]).validate_python(
|
||||
compacted_request.extra_headers
|
||||
)
|
||||
except Exception as e:
|
||||
if passthrough_on_no_deployment:
|
||||
return await original_generic_function(model=model, **kwargs)
|
||||
|
|
@ -5441,6 +5511,7 @@ class Router:
|
|||
# bucket, so the post-call carry-over below always has somewhere to read and write.
|
||||
kwargs.setdefault("litellm_metadata", {}) # mutable-ok: shared bucket # rebind-ok: stamp must be readable here
|
||||
|
||||
kwargs[COMPACTION_STATE_KEY] = CompactionState() # rebind-ok: share trusted request state with the snapshot
|
||||
fallback_kwargs: Final[dict[str, object]] = kwargs.copy()
|
||||
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
|
||||
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
|
||||
|
|
@ -5764,6 +5835,7 @@ class Router:
|
|||
# bucket, so the post-call carry-over below always has somewhere to read and write.
|
||||
kwargs.setdefault("litellm_metadata", {}) # mutable-ok: shared bucket # rebind-ok: stamp must be readable here
|
||||
|
||||
kwargs[COMPACTION_STATE_KEY] = CompactionState() # rebind-ok: share trusted request state with the snapshot
|
||||
fallback_kwargs: Final[dict[str, object]] = kwargs.copy() # mutable-ok: mutated below before re-entry
|
||||
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
|
||||
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
|
||||
|
|
@ -12278,11 +12350,231 @@ class Router:
|
|||
client = self.cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span)
|
||||
return client
|
||||
|
||||
def _compaction_model_budget(
|
||||
self,
|
||||
deployment: DeploymentTypedDict,
|
||||
request_defaults: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> ModelBudget | CompactionFailure:
|
||||
try:
|
||||
info: Final = self.get_router_model_info(
|
||||
deployment=TypeAdapter(dict[str, object]).validate_python(deployment),
|
||||
received_model_name=deployment["model_name"],
|
||||
)
|
||||
except Exception: # noqa: BLE001 # model-map resolution failures are explicit compaction failures
|
||||
return CompactionFailure("Context compaction requires known token limits for the selected deployment")
|
||||
input_limit: Final = info.get("max_input_tokens")
|
||||
output_limit: Final = info.get("max_output_tokens")
|
||||
if type(input_limit) is not int or type(output_limit) is not int or min(input_limit, output_limit) <= 0:
|
||||
return CompactionFailure(
|
||||
"Context compaction requires positive input and output limits for the selected deployment"
|
||||
)
|
||||
wire_model: Final = deployment["litellm_params"].get("model")
|
||||
if not isinstance(wire_model, str):
|
||||
return CompactionFailure("Context compaction requires a concrete deployment model")
|
||||
return ModelBudget(
|
||||
wire_model,
|
||||
input_limit,
|
||||
output_limit,
|
||||
request_defaults=request_defaults,
|
||||
deployment_id=_compaction_deployment_id(deployment),
|
||||
)
|
||||
|
||||
async def _acount_compaction_tokens(self, model: str, payload: Mapping[str, object]) -> int:
|
||||
messages: Final = payload.get("messages")
|
||||
input_value: Final = payload.get("input")
|
||||
instructions: Final = payload.get("instructions")
|
||||
return await offload_token_count(self._count_pre_call_check_tokens)(
|
||||
model=model,
|
||||
messages=TypeAdapter(list[dict[str, object]]).validate_python(
|
||||
(
|
||||
*(
|
||||
(MappingProxyType({"role": "system", "content": instructions}),)
|
||||
if isinstance(instructions, str) and instructions
|
||||
else ()
|
||||
),
|
||||
*messages,
|
||||
)
|
||||
)
|
||||
if isinstance(messages, (list, tuple))
|
||||
else None,
|
||||
input=TypeAdapter(str | list[object]).validate_python(input_value)
|
||||
if isinstance(input_value, (str, list))
|
||||
else None,
|
||||
request_kwargs=payload,
|
||||
)
|
||||
|
||||
def _compaction_request_payload(
|
||||
self,
|
||||
deployment: DeploymentTypedDict,
|
||||
request_kwargs: Mapping[str, object],
|
||||
messages: Sequence[Mapping[str, object]] | None = None,
|
||||
input: str | Sequence[object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
deployment_params: Final = TypeAdapter(dict[str, object]).validate_python(deployment)
|
||||
overridden_marker_keys: Final = self._forwarded_alias_marker_keys_the_deployment_sets(
|
||||
deployment=deployment_params,
|
||||
forwarded_keys=request_kwargs.get(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()),
|
||||
)
|
||||
effective_kwargs: Final = TypeAdapter(dict[str, object]).validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
**(MappingProxyType({"messages": messages}) if messages is not None else MappingProxyType({})),
|
||||
**(MappingProxyType({"input": input}) if input is not None else MappingProxyType({})),
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in request_kwargs.items() if key not in overridden_marker_keys}
|
||||
),
|
||||
"metadata": TypeAdapter(dict[str, object]).validate_python(
|
||||
request_kwargs.get("metadata") or MappingProxyType({})
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
self._merge_tools_from_deployment(deployment=deployment_params, kwargs=effective_kwargs)
|
||||
self._update_kwargs_with_default_litellm_params(kwargs=effective_kwargs)
|
||||
return MappingProxyType({**deployment["litellm_params"], **effective_kwargs})
|
||||
|
||||
async def _aprepare_selected_deployment(
|
||||
self,
|
||||
deployment: DeploymentTypedDict,
|
||||
model: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
messages: Sequence[Mapping[str, object]] | None = None,
|
||||
input: str | Sequence[object] | None = None,
|
||||
protocol: Literal["chat", "responses", "messages"] = "chat",
|
||||
) -> CompactedRequest | None:
|
||||
state: Final = compaction_state(request_kwargs)
|
||||
native_request: Final = current_native_request()
|
||||
if native_request is None and (state is None or state.model is None):
|
||||
return None
|
||||
payload: Final = self._compaction_request_payload(deployment, request_kwargs, messages, input)
|
||||
timeout: Final = self._get_non_stream_timeout(
|
||||
kwargs=TypeAdapter(dict[str, object]).validate_python(request_kwargs),
|
||||
data=TypeAdapter(dict[str, object]).validate_python(deployment["litellm_params"]),
|
||||
)
|
||||
if state is not None and isinstance(timeout, (int, float)):
|
||||
state.limit_timeout(float(timeout))
|
||||
target: Final = self._compaction_model_budget(deployment, request_defaults=payload)
|
||||
if isinstance(target, CompactionFailure):
|
||||
raise_compaction_failure(target, model)
|
||||
if native_request is not None:
|
||||
native_failure: Final = await validate_native_recipient(
|
||||
native_request, target, payload, float(timeout) if isinstance(timeout, (int, float)) else 120.0
|
||||
)
|
||||
if native_failure is not None:
|
||||
raise_compaction_failure(native_failure, model)
|
||||
return None
|
||||
if state is None or state.model is None:
|
||||
return None
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
effective_turn_off_message_logging,
|
||||
forwarded_internal_call_metadata,
|
||||
parent_session_kwargs,
|
||||
)
|
||||
|
||||
metadata_source: Final = MappingProxyType(
|
||||
{
|
||||
key: TypeAdapter(dict[str, object]).validate_python(request_kwargs[key])
|
||||
for key in ("metadata", "litellm_metadata")
|
||||
if isinstance(request_kwargs.get(key), Mapping)
|
||||
}
|
||||
)
|
||||
summary_metadata: Final = MappingProxyType(
|
||||
forwarded_internal_call_metadata(
|
||||
TypeAdapter(dict[str, object]).validate_python(
|
||||
get_litellm_metadata_from_kwargs(
|
||||
TypeAdapter(dict[str, object]).validate_python(
|
||||
MappingProxyType({"litellm_params": metadata_source})
|
||||
)
|
||||
)
|
||||
),
|
||||
"context_compaction",
|
||||
)
|
||||
)
|
||||
summary_request: Final = MappingProxyType({"metadata": summary_metadata})
|
||||
|
||||
async def sdk_summary(model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]:
|
||||
if request_kwargs.get("proxy_server_request") is not None:
|
||||
raise_compaction_failure(
|
||||
CompactionFailure("The authenticated compaction executor is unavailable"), model
|
||||
)
|
||||
native_kwargs: Final = MappingProxyType(
|
||||
{
|
||||
**request.payload,
|
||||
**parent_session_kwargs(request_kwargs),
|
||||
**MappingProxyType(
|
||||
{key: request_kwargs[key] for key in ("allowed_model_region", "tags") if key in request_kwargs}
|
||||
),
|
||||
"model": model,
|
||||
**(MappingProxyType({"stream": False}) if request.protocol == "messages" else MappingProxyType({})),
|
||||
"timeout": timeout,
|
||||
"num_retries": 0,
|
||||
"max_retries": 0,
|
||||
"disable_fallbacks": True,
|
||||
"metadata": TypeAdapter(dict[str, object]).validate_python(summary_metadata),
|
||||
"turn_off_message_logging": effective_turn_off_message_logging(request_kwargs),
|
||||
}
|
||||
)
|
||||
response: Final[object] = (
|
||||
await self.acompact_responses(**native_kwargs)
|
||||
if request.protocol == "responses"
|
||||
else await self.aanthropic_messages(**native_kwargs)
|
||||
)
|
||||
return TypeAdapter(dict[str, object]).validate_python(
|
||||
response.model_dump(exclude_none=True) if isinstance(response, BaseModel) else response
|
||||
)
|
||||
|
||||
candidates: Final = cast( # cast-ok: access-group owner only filters Router-validated deployments
|
||||
tuple[DeploymentTypedDict, ...],
|
||||
tuple(
|
||||
self._filter_deployments_by_model_access_groups(
|
||||
model=self._get_model_from_alias(state.model) or state.model,
|
||||
healthy_deployments=TypeAdapter(list[object]).validate_python(
|
||||
self.deployments_for_request(state.model, summary_request)
|
||||
),
|
||||
request_kwargs=TypeAdapter(dict[str, object]).validate_python(summary_request),
|
||||
request_team_id=get_request_team_id(summary_request),
|
||||
),
|
||||
),
|
||||
)
|
||||
budgets: Final = tuple(
|
||||
self._compaction_model_budget(
|
||||
candidate,
|
||||
request_defaults=self._compaction_request_payload(
|
||||
candidate,
|
||||
MappingProxyType(
|
||||
{
|
||||
"messages": (),
|
||||
"stream": False,
|
||||
"num_retries": 0,
|
||||
"max_retries": 0,
|
||||
"disable_fallbacks": True,
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
for candidate in candidates
|
||||
if not self._is_strategy_marker_deployment(candidate)
|
||||
)
|
||||
summary_budgets: Final = tuple(budget for budget in budgets if isinstance(budget, ModelBudget))
|
||||
prepared: Final = await prepare_compaction(
|
||||
payload=payload,
|
||||
target=target,
|
||||
state=state,
|
||||
summary_budgets=summary_budgets if len(summary_budgets) == len(candidates) else (),
|
||||
executor=current_summary_executor() or sdk_summary,
|
||||
counter=self._acount_compaction_tokens,
|
||||
protocol=protocol,
|
||||
)
|
||||
if isinstance(prepared, CompactionFailure):
|
||||
raise_compaction_failure(prepared, model)
|
||||
return prepared
|
||||
|
||||
def _count_pre_call_check_tokens(
|
||||
self,
|
||||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
messages: Sequence[Mapping[str, object]] | None,
|
||||
input: str | Sequence[object] | None,
|
||||
request_kwargs: Mapping[str, object] | None = None,
|
||||
model: str = "",
|
||||
) -> int:
|
||||
"""
|
||||
Count input tokens for context-window pre-call checks.
|
||||
|
|
@ -12301,7 +12593,7 @@ class Router:
|
|||
anthropic_system_to_openai_message,
|
||||
)
|
||||
|
||||
extras: Final = request_kwargs if request_kwargs is not None else MappingProxyType({})
|
||||
extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({})
|
||||
raw_instructions: Final = extras.get("instructions")
|
||||
instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None
|
||||
raw_tools: Final = extras.get("tools")
|
||||
|
|
@ -12313,7 +12605,7 @@ class Router:
|
|||
system_message: Final = anthropic_system_to_openai_message(extras.get("system"))
|
||||
if messages is not None:
|
||||
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
|
||||
return litellm.token_counter(messages=counted_messages, tools=tools)
|
||||
return litellm.token_counter(model=model, messages=counted_messages, tools=tools)
|
||||
if input is not None:
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
|
|
@ -12327,6 +12619,7 @@ class Router:
|
|||
responses_api_request={"instructions": instructions} if instructions is not None else {},
|
||||
)
|
||||
return litellm.token_counter(
|
||||
model=model,
|
||||
messages=cast(list, input_messages), # cast-ok: transformed chat messages
|
||||
tools=tools,
|
||||
)
|
||||
|
|
@ -12452,7 +12745,11 @@ class Router:
|
|||
model_info = self.get_router_model_info(deployment=deployment, received_model_name=model)
|
||||
|
||||
max_input_tokens = model_info.get("max_input_tokens") if isinstance(model_info, dict) else None
|
||||
if isinstance(max_input_tokens, int) and has_countable_input:
|
||||
if (
|
||||
isinstance(max_input_tokens, int)
|
||||
and has_countable_input
|
||||
and not defers_context_filter(request_kwargs)
|
||||
):
|
||||
if input_tokens is None:
|
||||
if skip_inline_token_count:
|
||||
return _returned_deployments
|
||||
|
|
@ -12745,6 +13042,22 @@ class Router:
|
|||
_access_group_filter_emptied_candidates = (
|
||||
_pre_model_access_group_filter_len > 0 and len(healthy_deployments) == 0
|
||||
)
|
||||
native_request: Final = current_native_request()
|
||||
if native_request is not None:
|
||||
native_deployments: Final = cast( # cast-ok: normal admission filters Router-validated deployments
|
||||
Sequence[DeploymentTypedDict], healthy_deployments
|
||||
)
|
||||
healthy_deployments = TypeAdapter(
|
||||
list[dict[str, object]]
|
||||
).validate_python( # rebind-ok: constrain normal admission
|
||||
tuple(
|
||||
deployment
|
||||
for deployment in native_deployments
|
||||
if _compaction_deployment_id(deployment) in native_request.allowed_deployment_ids
|
||||
)
|
||||
)
|
||||
if not healthy_deployments:
|
||||
raise_compaction_failure(CompactionFailure("No eligible native compactor remains available"), model)
|
||||
|
||||
if len(healthy_deployments) == 0:
|
||||
# check if the user sent in a deployment name instead
|
||||
|
|
@ -13174,6 +13487,23 @@ class Router:
|
|||
and self.routing_strategy != "latency-based-routing"
|
||||
and self.routing_strategy != "least-busy"
|
||||
): # prevent regressions for other routing strategies, that don't have async get available deployments implemented.
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
|
||||
legacy_strategy: Final = self._select_pre_routing_strategy(
|
||||
model=self._get_model_from_alias(model) or model, request_kwargs=request_kwargs
|
||||
)
|
||||
if (
|
||||
legacy_strategy is not None
|
||||
and isinstance(legacy_strategy.strategy, ComplexityRouter)
|
||||
and legacy_strategy.strategy.config.context_window_compaction_model is not None
|
||||
):
|
||||
raise_compaction_failure(
|
||||
CompactionFailure(
|
||||
"Context compaction is unsupported with legacy synchronous deployment selection; "
|
||||
"use simple-shuffle or usage-based-routing-v2"
|
||||
),
|
||||
model,
|
||||
)
|
||||
return self.get_available_deployment(
|
||||
model=model,
|
||||
messages=messages,
|
||||
|
|
@ -13704,6 +14034,16 @@ class Router:
|
|||
selected_strategy: Final = self._select_pre_routing_strategy(
|
||||
model=registered_model_name, request_kwargs=request_kwargs
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
|
||||
context_state: Final = compaction_state(request_kwargs)
|
||||
if (
|
||||
context_state is not None
|
||||
and selected_strategy is not None
|
||||
and isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
and selected_strategy.strategy.config.context_window_compaction_model is not None
|
||||
):
|
||||
context_state.arm(selected_strategy.strategy.config.context_window_compaction_model)
|
||||
if selected_strategy is None:
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
self._stamp_or_clear_metadata_key(
|
||||
|
|
|
|||
|
|
@ -362,6 +362,60 @@ response = litellm.completion(
|
|||
|
||||
## Special Behaviors
|
||||
|
||||
### Compact history on context overflow
|
||||
|
||||
Set `context_window_compaction_model` to an ordinary Router model group that supports native
|
||||
compaction and has enough context for the input. It receives older conversation history, so choose
|
||||
a recipient appropriate for that data. The group must already exist in `model_list`; another Auto
|
||||
Router cannot serve as the compaction model
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: smart-router
|
||||
litellm_params:
|
||||
model: auto_router/complexity_router
|
||||
complexity_router_config:
|
||||
tiers:
|
||||
SIMPLE: small-model
|
||||
COMPLEX: large-model
|
||||
context_window_compaction_model: conversation-summary
|
||||
```
|
||||
|
||||
Here `small-model`, `large-model`, and `conversation-summary` refer to your configured model groups.
|
||||
Omitting the setting or using `null` keeps existing behavior. Empty and whitespace-only names are
|
||||
rejected. When configured, compaction takes precedence over `enable_context_window_escalation`
|
||||
|
||||
Supported overflow routes are asynchronous Responses with native OpenAI deployments and Messages
|
||||
with native Anthropic deployments, including streaming answers. OpenAI uses `/responses/compact`;
|
||||
Anthropic uses on-demand compaction with the `compact-2026-09-04` beta. The producer must support
|
||||
that provider's compaction API. Native state is not translated between providers or APIs
|
||||
|
||||
The answering deployment, including its session pin, stays unchanged. Completed older history goes
|
||||
to the native compactor; the newest actual user request and its active tool tail stay untouched.
|
||||
OpenAI's complete returned window or Anthropic's exact signed block is preserved, then the target's
|
||||
native token-count endpoint checks compatibility and fit before answering. Requests that already
|
||||
fit use their original history. Unsupported overflow routes, including Chat Completions, fail
|
||||
explicitly rather than falling back to a custom summary. Synchronous SDK calls and raw passthrough
|
||||
are outside this feature. The legacy `usage-based-routing` selector is unsupported; use an
|
||||
async-native selector such as `simple-shuffle` or `usage-based-routing-v2`
|
||||
|
||||
Native compaction can omit details. It adds latency and a separately billable model call. There is
|
||||
no exact upfront cost promise, automatic cost comparison, or guarantee it costs less than choosing
|
||||
a larger model. The caller needs normal access to the configured compactor; its call remains
|
||||
subject to normal budgets, rate limits, deployment restrictions, and spend accounting
|
||||
|
||||
The input budget reserves the effective output limit and a safety margin. Omitting the output limit
|
||||
conservatively reserves the deployment's maximum output capability. Only one compaction call is
|
||||
allowed within 120 seconds, further bounded by the request timeout. Anthropic compaction requests
|
||||
reserve 4,096 output tokens; OpenAI's compact endpoint does not accept an output-token cap. Retries
|
||||
can reuse the native result within the same request. A delayed streaming fallback cannot start new
|
||||
proxy compaction calls after the initial response scope closes
|
||||
|
||||
If there is no eligible compactor, no closed older prefix, unavailable `previous_response_id`
|
||||
history, or incompatible or oversized native output, the request fails with an explicit context
|
||||
error. Token-count failures never fall back to a local estimate for native-state validation.
|
||||
Original caller history is retained on failure; nothing is silently truncated
|
||||
|
||||
### Modality-based capability routing
|
||||
|
||||
The classifier reads text alone, so a request carrying an image can classify cheap and land on a
|
||||
|
|
|
|||
|
|
@ -3099,7 +3099,11 @@ class ComplexityRouter(CustomLogger):
|
|||
resolved_messages: Sequence[Mapping[str, object]] | None,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> _RequestContextFit:
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
if (
|
||||
self.config.context_window_compaction_model is not None
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer)
|
||||
names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset(
|
||||
(self.config.default_model,) if self.config.default_model else ()
|
||||
|
|
@ -3125,7 +3129,11 @@ class ComplexityRouter(CustomLogger):
|
|||
(the placement stands). Only a real tokenizer count ever moves a request, escalation
|
||||
lands only on groups whose every deployment declares a fitting window, and a group
|
||||
with no resolvable window is never moved on faith in either direction."""
|
||||
if not self.config.enable_context_window_escalation or not resolved_messages:
|
||||
if (
|
||||
self.config.context_window_compaction_model is not None
|
||||
or not self.config.enable_context_window_escalation
|
||||
or not resolved_messages
|
||||
):
|
||||
return None
|
||||
pools: Final = self._tier_pools()
|
||||
pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))
|
||||
|
|
|
|||
|
|
@ -1254,6 +1254,20 @@ class ComplexityRouterConfig(BaseModel):
|
|||
),
|
||||
)
|
||||
|
||||
context_window_compaction_model: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
pattern=r"\S",
|
||||
description=(
|
||||
"Ordinary Router model group used for provider-native compaction when an asynchronous "
|
||||
"Responses/OpenAI or Messages/Anthropic request exceeds its selected deployment's "
|
||||
"input budget. The compactor must fit the input and its native output must fit the "
|
||||
"unchanged answering deployment. Unsupported overflow routes fail explicitly. "
|
||||
"Compaction takes precedence over context window escalation and incurs a separate "
|
||||
"billable call under normal access and limits. Nested Auto Routers are not supported. "
|
||||
"Omit or set null to keep existing routing behavior."
|
||||
),
|
||||
)
|
||||
enable_context_window_escalation: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
|
|
|
|||
403
litellm/router_strategy/complexity_router/context_compaction.py
Normal file
403
litellm/router_strategy/complexity_router/context_compaction.py
Normal file
|
|
@ -0,0 +1,403 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, NoReturn, Protocol
|
||||
|
||||
import anyio
|
||||
from pydantic import StrictStr, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.litellm_core_utils.prompt_templates.compaction import (
|
||||
CompactionHistoryError,
|
||||
NativeProtocol,
|
||||
compaction_headers,
|
||||
native_compaction_payload,
|
||||
native_compaction_result,
|
||||
split_native_history,
|
||||
)
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.openai.responses.count_tokens.handler import OpenAICountTokensHandler
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import ProviderSpecificHeader
|
||||
|
||||
COMPACTION_STATE_KEY: Final = "_context_window_compaction_state"
|
||||
MAX_COMPACTION_SECONDS: Final = 120.0
|
||||
SUMMARY_OUTPUT_TOKENS: Final = 4096
|
||||
_PROTECTED_BODY_FIELDS: Final = frozenset(
|
||||
{
|
||||
"model",
|
||||
"messages",
|
||||
"input",
|
||||
"prompt",
|
||||
"instructions",
|
||||
"system",
|
||||
"tools",
|
||||
"functions",
|
||||
"tool_choice",
|
||||
"function_call",
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"max_output_tokens",
|
||||
"thinking",
|
||||
"reasoning",
|
||||
"context_management",
|
||||
"compaction",
|
||||
"previous_response_id",
|
||||
"conversation",
|
||||
}
|
||||
)
|
||||
_HEADERS: Final = TypeAdapter(dict[StrictStr, StrictStr])
|
||||
_COUNT_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None]] = (
|
||||
TypeAdapter(ProviderSpecificHeader | tuple[ProviderSpecificHeader, ...] | None)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompactionFailure:
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelBudget:
|
||||
model: str
|
||||
input_limit: int
|
||||
output_limit: int
|
||||
request_defaults: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
|
||||
deployment_id: str = ""
|
||||
|
||||
def input_budget(self, output_tokens: int) -> int:
|
||||
return self.input_limit - output_tokens - max(64, self.input_limit // 100)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompactedRequest:
|
||||
field: str
|
||||
value: Sequence[Mapping[str, object]]
|
||||
extra_headers: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeRequest:
|
||||
protocol: NativeProtocol
|
||||
payload: Mapping[str, object]
|
||||
allowed_deployment_ids: frozenset[str]
|
||||
closed: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SummaryMemo:
|
||||
identity: str
|
||||
response: Mapping[str, object]
|
||||
|
||||
|
||||
class SummaryExecutor(Protocol):
|
||||
async def __call__(self, model: str, request: NativeRequest, timeout: float) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
class TokenCounter(Protocol):
|
||||
async def __call__(self, model: str, payload: Mapping[str, object]) -> int: ...
|
||||
|
||||
|
||||
class NativeTokenCounter(Protocol):
|
||||
async def __call__(
|
||||
self, target: ModelBudget, payload: Mapping[str, object], protocol: NativeProtocol, timeout: float
|
||||
) -> int: ...
|
||||
|
||||
|
||||
_summary_executor: Final[ContextVar[SummaryExecutor | None]] = ContextVar("context_compaction_executor", default=None)
|
||||
_native_request: Final[ContextVar[NativeRequest | None]] = ContextVar("native_compaction_request", default=None)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def use_summary_executor(executor: SummaryExecutor) -> Generator[None]:
|
||||
token: Final = _summary_executor.set(executor)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_summary_executor.reset(token)
|
||||
|
||||
|
||||
def current_summary_executor() -> SummaryExecutor | None:
|
||||
return _summary_executor.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def native_request_scope(request: NativeRequest) -> Generator[None]:
|
||||
token: Final = _native_request.set(request)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
request.closed.set()
|
||||
_native_request.reset(token)
|
||||
|
||||
|
||||
def current_native_request() -> NativeRequest | None:
|
||||
request: Final = _native_request.get()
|
||||
return request if request is not None and not request.closed.is_set() else None
|
||||
|
||||
|
||||
class CompactionState:
|
||||
def __init__(self, timeout: float = MAX_COMPACTION_SECONDS) -> None:
|
||||
self.model: str | None = None
|
||||
self.deadline: float = time.monotonic() + min(timeout, MAX_COMPACTION_SECONDS)
|
||||
self.calls: int = 0
|
||||
self.memo: SummaryMemo | None = None
|
||||
self.failure: CompactionFailure | None = None
|
||||
|
||||
def arm(self, model: str) -> None:
|
||||
self.model = model
|
||||
|
||||
def remaining(self) -> float:
|
||||
return max(0.0, self.deadline - time.monotonic())
|
||||
|
||||
def limit_timeout(self, timeout: float) -> None:
|
||||
self.deadline = min(self.deadline, time.monotonic() + timeout)
|
||||
|
||||
def failed(self, failure: CompactionFailure) -> CompactionFailure:
|
||||
self.failure = failure
|
||||
return failure
|
||||
|
||||
async def compact(self, request: NativeRequest, identity: str, executor: SummaryExecutor) -> Mapping[str, object]:
|
||||
if self.memo is not None and self.memo.identity == identity:
|
||||
return self.memo.response
|
||||
if self.model is None or self.calls or self.remaining() <= 0:
|
||||
raise ValueError("Native compaction exhausted its call or time budget")
|
||||
self.calls += 1
|
||||
with native_request_scope(request), anyio.fail_after(self.remaining()):
|
||||
response: Final = await executor(self.model, request, self.remaining())
|
||||
self.memo = SummaryMemo(identity, response)
|
||||
return response
|
||||
|
||||
|
||||
def compaction_state(payload: Mapping[str, object] | None) -> CompactionState | None:
|
||||
candidate: Final = payload.get(COMPACTION_STATE_KEY) if payload is not None else None
|
||||
return candidate if isinstance(candidate, CompactionState) else None
|
||||
|
||||
|
||||
def defers_context_filter(payload: Mapping[str, object] | None) -> bool:
|
||||
state: Final = compaction_state(payload)
|
||||
return state is not None and state.model is not None
|
||||
|
||||
|
||||
def raise_compaction_failure(failure: CompactionFailure, model: str) -> NoReturn:
|
||||
raise litellm.ContextWindowExceededError(message=failure.message, model=model, llm_provider="")
|
||||
|
||||
|
||||
def output_reservation(payload: Mapping[str, object], budget: ModelBudget) -> int | CompactionFailure:
|
||||
ceilings: Final = tuple(
|
||||
payload[key]
|
||||
for key in ("max_output_tokens", "max_completion_tokens", "max_tokens")
|
||||
if payload.get(key) is not None
|
||||
)
|
||||
if any(
|
||||
isinstance(value, bool) or not isinstance(value, int) or not 0 < value <= budget.output_limit
|
||||
for value in ceilings
|
||||
):
|
||||
return CompactionFailure("Native compaction requires valid output-token limits for the selected deployment")
|
||||
ceiling: Final = max(TypeAdapter(tuple[int, ...]).validate_python(ceilings), default=budget.output_limit)
|
||||
thinking: Final = payload.get("thinking")
|
||||
thinking_budget: Final = thinking.get("budget_tokens") if isinstance(thinking, Mapping) else None
|
||||
if thinking_budget is not None and (
|
||||
isinstance(thinking_budget, bool) or not isinstance(thinking_budget, int) or not 0 < thinking_budget < ceiling
|
||||
):
|
||||
return CompactionFailure("The selected output ceiling must exceed the thinking budget")
|
||||
return ceiling
|
||||
|
||||
|
||||
def validate_compaction_overrides(payload: Mapping[str, object]) -> CompactionFailure | None:
|
||||
body: Final = payload.get("extra_body")
|
||||
if body is not None and (not isinstance(body, Mapping) or not _PROTECTED_BODY_FIELDS.isdisjoint(body)):
|
||||
return CompactionFailure(
|
||||
"Native compaction does not allow extra_body to override history or token-budget fields"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def native_provider(model: str) -> tuple[str, str]:
|
||||
resolved: Final = litellm.get_llm_provider(model=model)
|
||||
return resolved[0].removeprefix("responses/"), resolved[1]
|
||||
|
||||
|
||||
def supports_native_protocol(model: str, protocol: str) -> bool:
|
||||
provider: Final = native_provider(model)[1]
|
||||
return (protocol == "responses" and provider == "openai") or (protocol == "messages" and provider == "anthropic")
|
||||
|
||||
|
||||
def _credential(value: object) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
return get_secret_str(value.removeprefix("os.environ/")) if value.startswith("os.environ/") else value
|
||||
|
||||
|
||||
async def count_native_tokens(
|
||||
target: ModelBudget, payload: Mapping[str, object], protocol: NativeProtocol, timeout: float
|
||||
) -> int:
|
||||
model, provider = native_provider(target.model)
|
||||
if (protocol, provider) not in (("responses", "openai"), ("messages", "anthropic")):
|
||||
raise ValueError("Native token counting requires a matching native provider")
|
||||
params: Final = target.request_defaults
|
||||
if any(source.get("client") is not None for source in (params, payload)) or any(
|
||||
source.get("custom_llm_provider") not in (None, provider) for source in (params, payload)
|
||||
):
|
||||
raise ValueError("Native token counting cannot verify a custom client or provider override")
|
||||
api_key: Final = _credential(params.get("api_key"))
|
||||
api_base: Final = _credential(params.get("api_base"))
|
||||
try:
|
||||
scoped_headers: Final = _HEADERS.validate_python(
|
||||
ProviderSpecificHeaderUtils.get_provider_specific_headers(
|
||||
provider_specific_header=_COUNT_SCOPED_HEADERS.validate_python(
|
||||
payload.get("provider_specific_header", params.get("provider_specific_header"))
|
||||
),
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
)
|
||||
header_sources: Final = (
|
||||
*(source.get(field) for field in ("headers", "extra_headers") for source in (params, payload)),
|
||||
scoped_headers,
|
||||
)
|
||||
headers: Final = MappingProxyType(
|
||||
{
|
||||
name.lower(): value
|
||||
for source in header_sources
|
||||
for name, value in _HEADERS.validate_python(
|
||||
source if source is not None else MappingProxyType({})
|
||||
).items()
|
||||
}
|
||||
)
|
||||
handler: Final = OpenAICountTokensHandler() if protocol == "responses" else AnthropicCountTokensHandler()
|
||||
return await handler.count_native_tokens(
|
||||
model=model, payload=payload, api_key=api_key, api_base=api_base, headers=headers, timeout=timeout
|
||||
)
|
||||
except ValueError:
|
||||
raise ValueError("Provider-native token counting failed") from None
|
||||
|
||||
|
||||
async def validate_native_recipient(
|
||||
request: NativeRequest,
|
||||
target: ModelBudget,
|
||||
payload: Mapping[str, object],
|
||||
timeout: float,
|
||||
counter: NativeTokenCounter = count_native_tokens,
|
||||
) -> CompactionFailure | None:
|
||||
if target.deployment_id not in request.allowed_deployment_ids or not supports_native_protocol(
|
||||
target.model, request.protocol
|
||||
):
|
||||
return CompactionFailure("Native compaction selected an incompatible deployment")
|
||||
invalid: Final = validate_compaction_overrides(payload)
|
||||
if invalid is not None:
|
||||
return invalid
|
||||
for key in ("input", "messages", "instructions", "system", "tools", "compaction"):
|
||||
if payload.get(key) != request.payload.get(key):
|
||||
return CompactionFailure("Compactor deployment defaults override the native request")
|
||||
reserve: Final = SUMMARY_OUTPUT_TOKENS if request.protocol == "messages" else target.output_limit
|
||||
if request.protocol == "messages" and payload.get("max_tokens") != SUMMARY_OUTPUT_TOKENS:
|
||||
return CompactionFailure("Compactor deployment overrides the native output allowance")
|
||||
count: Final = await counter(target, payload, request.protocol, timeout)
|
||||
if type(count) is not int or not 0 <= count <= target.input_budget(reserve):
|
||||
return CompactionFailure("The selected native compactor cannot fit the complete request")
|
||||
return None
|
||||
|
||||
|
||||
def _has_native_state(payload: Mapping[str, object]) -> bool:
|
||||
items: Final = payload.get("input") or payload.get("messages")
|
||||
if not isinstance(items, (list, tuple)):
|
||||
return False
|
||||
return any(
|
||||
isinstance(item, Mapping)
|
||||
and (
|
||||
item.get("type") in ("compaction", "reasoning", "item_reference")
|
||||
or isinstance(item.get("content"), (list, tuple))
|
||||
and any(isinstance(block, Mapping) and block.get("type") == "compaction" for block in item["content"])
|
||||
)
|
||||
for item in items
|
||||
)
|
||||
|
||||
|
||||
async def prepare_compaction(
|
||||
payload: Mapping[str, object],
|
||||
target: ModelBudget,
|
||||
state: CompactionState,
|
||||
summary_budgets: tuple[ModelBudget, ...],
|
||||
executor: SummaryExecutor,
|
||||
counter: TokenCounter,
|
||||
protocol: Literal["chat", "responses", "messages"] = "chat",
|
||||
native_counter: NativeTokenCounter = count_native_tokens,
|
||||
) -> CompactedRequest | CompactionFailure | None:
|
||||
invalid: Final = validate_compaction_overrides(payload)
|
||||
if invalid is not None:
|
||||
return invalid
|
||||
reserve: Final = output_reservation(payload, target)
|
||||
if isinstance(reserve, CompactionFailure):
|
||||
return reserve
|
||||
input_budget: Final = target.input_budget(reserve)
|
||||
try:
|
||||
if not _has_native_state(payload) and await counter(target.model, payload) <= input_budget:
|
||||
return None
|
||||
if protocol == "chat" or not supports_native_protocol(target.model, protocol):
|
||||
return CompactionFailure(
|
||||
"Native compaction requires Responses/OpenAI or Messages/Anthropic; this route is unsupported"
|
||||
)
|
||||
if payload.get("previous_response_id") is not None or payload.get("conversation") is not None:
|
||||
return CompactionFailure("Native overflow compaction requires the complete conversation history")
|
||||
with anyio.fail_after(state.remaining()):
|
||||
input_tokens: Final = await native_counter(target, payload, protocol, state.remaining())
|
||||
if type(input_tokens) is not int or input_tokens < 0:
|
||||
return CompactionFailure("Native token counting returned an invalid input count")
|
||||
if input_tokens <= input_budget:
|
||||
return None
|
||||
history: Final = split_native_history(payload, protocol)
|
||||
if isinstance(history, CompactionHistoryError):
|
||||
return CompactionFailure(history.message)
|
||||
eligible: Final = tuple(
|
||||
budget
|
||||
for budget in summary_budgets
|
||||
if budget.deployment_id
|
||||
and supports_native_protocol(budget.model, protocol)
|
||||
and budget.output_limit >= SUMMARY_OUTPUT_TOKENS
|
||||
and input_tokens
|
||||
<= budget.input_budget(SUMMARY_OUTPUT_TOKENS if protocol == "messages" else budget.output_limit)
|
||||
)
|
||||
if not eligible:
|
||||
return CompactionFailure("No compatible configured native compactor can fit the complete input")
|
||||
body: Final = native_compaction_payload(history, payload, protocol)
|
||||
request: Final = NativeRequest(protocol, body, frozenset(budget.deployment_id for budget in eligible))
|
||||
identity: Final = hashlib.sha256(
|
||||
json.dumps(
|
||||
MappingProxyType(
|
||||
{"model": state.model, "protocol": protocol, "payload": body, "tools": payload.get("tools")}
|
||||
),
|
||||
sort_keys=True,
|
||||
default=dict,
|
||||
).encode()
|
||||
).hexdigest()
|
||||
response: Final = await state.compact(request, identity, executor)
|
||||
result: Final = native_compaction_result(response, history, protocol)
|
||||
if isinstance(result, CompactionHistoryError):
|
||||
return CompactionFailure(result.message)
|
||||
headers: Final = (
|
||||
compaction_headers(payload.get("extra_headers")) if protocol == "messages" else MappingProxyType({})
|
||||
)
|
||||
prepared: Final = MappingProxyType(
|
||||
{
|
||||
**payload,
|
||||
history.field: result,
|
||||
"extra_headers": headers or payload.get("extra_headers") or MappingProxyType({}),
|
||||
}
|
||||
)
|
||||
final_count: Final = await native_counter(target, prepared, protocol, state.remaining())
|
||||
if type(final_count) is not int or not 0 <= final_count <= input_budget:
|
||||
return CompactionFailure("Native compaction output does not fit the pinned target")
|
||||
return CompactedRequest(history.field, result, headers)
|
||||
except Exception as error: # noqa: BLE001 # provider failures become sanitized context errors at this boundary
|
||||
return state.failed(
|
||||
CompactionFailure(f"Native compaction failed ({type(error).__name__}); original history retained")
|
||||
)
|
||||
|
|
@ -24,7 +24,7 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
|
|||
|
||||
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
|
||||
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding"]
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "compaction"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -154,6 +154,7 @@ def strategy_router_dependencies(
|
|||
dict.fromkeys(
|
||||
tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier"))
|
||||
+ _named(litellm_params.get("complexity_router_default_model"), "default")
|
||||
+ _named(complexity.get("context_window_compaction_model"), "compaction")
|
||||
+ (
|
||||
_named(classifier.get("model"), "classifier")
|
||||
if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES
|
||||
|
|
|
|||
|
|
@ -389,6 +389,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = (
|
|||
)
|
||||
|
||||
|
||||
class AnthropicCompactionRequest(TypedDict):
|
||||
type: ReadOnly[Literal["summarize"]]
|
||||
instructions: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
||||
max_tokens: int | None
|
||||
metadata: AnthropicMetadata | dict | None
|
||||
|
|
@ -410,6 +415,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
|||
output_config: AnthropicOutputConfig | None # Configuration for Claude's output behavior
|
||||
cache_control: dict[str, Any] | None # Automatic prompt caching
|
||||
reasoning_effort: str | None
|
||||
compaction: ReadOnly[AnthropicCompactionRequest | None]
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False):
|
||||
|
|
@ -568,6 +574,7 @@ class CompactionBlock(TypedDict, total=False):
|
|||
|
||||
type: Required[Literal["compaction"]]
|
||||
content: str | None
|
||||
signature: ReadOnly[str]
|
||||
|
||||
|
||||
class UsageIteration(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -2954,6 +2954,7 @@ RoutingDecisionCause = Literal[
|
|||
|
||||
|
||||
InternalCallOrigin = Literal[
|
||||
"context_compaction",
|
||||
"autorouter_classifier",
|
||||
"shadow_eval_router",
|
||||
"shadow_eval_judge",
|
||||
|
|
@ -2964,6 +2965,7 @@ InternalCallOrigin = Literal[
|
|||
records that it is not traffic the caller sent."""
|
||||
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN: Final[InternalCallOrigin] = "autorouter_classifier"
|
||||
CONTEXT_COMPACTION_CALL_ORIGIN: Final[InternalCallOrigin] = "context_compaction"
|
||||
SHADOW_EVAL_ROUTER_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_router"
|
||||
SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
|
||||
LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN: Final[InternalCallOrigin] = "llm_as_a_judge_guardrail"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,148 @@
|
|||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.compaction import (
|
||||
CompactionHistoryError,
|
||||
NativeHistory,
|
||||
NativeProtocol,
|
||||
compaction_headers,
|
||||
native_compaction_payload,
|
||||
native_compaction_result,
|
||||
split_native_history,
|
||||
)
|
||||
|
||||
_USER: Final = {"role": "user", "content": "latest"}
|
||||
_HISTORY: Final = NativeHistory("input", (), (_USER,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
def test_native_payload_preserves_source_and_only_forwards_supported_parameters(protocol: NativeProtocol) -> None:
|
||||
field: Final = "input" if protocol == "responses" else "messages"
|
||||
older: Final = {"role": "user", "content": [{"type": "input_file", "file_id": "opaque-file"}]}
|
||||
state: Final = {"type": "compaction", "encrypted_content": "opaque"}
|
||||
payload: Final = {
|
||||
field: [older, state, _USER],
|
||||
"instructions": "rules",
|
||||
"system": "system",
|
||||
"tools": [{"name": "tool"}],
|
||||
"temperature": 1,
|
||||
"max_tokens": 1,
|
||||
"extra_headers": {"Anthropic-Beta": "other", "x-test": "keep"},
|
||||
}
|
||||
original: Final = deepcopy(payload)
|
||||
history: Final = split_native_history(payload, protocol)
|
||||
assert isinstance(history, NativeHistory)
|
||||
assert (history.field, history.prefix, history.tail) == (field, (older, state), (_USER,))
|
||||
request: Final = native_compaction_payload(history, payload, protocol)
|
||||
assert request[field] == [older, state]
|
||||
if protocol == "responses":
|
||||
assert request == {"input": [older, state], "instructions": "rules"}
|
||||
else:
|
||||
assert request == {
|
||||
"messages": [older, state],
|
||||
"system": "system",
|
||||
"tools": [{"name": "tool"}],
|
||||
"compaction": {"type": "summarize"},
|
||||
"max_tokens": 4096,
|
||||
"extra_headers": compaction_headers(payload["extra_headers"]),
|
||||
}
|
||||
request[field][0]["content"][0]["file_id"] = "changed-private-copy"
|
||||
assert payload == original and history.prefix[0] == older
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mixed", (False, True))
|
||||
def test_messages_tool_result_is_not_a_new_request(mixed: bool) -> None:
|
||||
older: Final = {"role": "user", "content": "old"}
|
||||
call: Final = {"role": "assistant", "content": [{"type": "tool_use", "id": "call"}]}
|
||||
block: Final = {"type": "tool_result", "tool_use_id": "call", "content": "ok"}
|
||||
result: Final = {"role": "user", "content": [block, *([{"type": "text", "text": "next"}] if mixed else [])]}
|
||||
history: Final = split_native_history({"messages": [older, _USER, call, result]}, "messages")
|
||||
assert isinstance(history, NativeHistory)
|
||||
assert history.prefix == ((older, _USER, call) if mixed else (older,))
|
||||
assert history.tail == ((result,) if mixed else (_USER, call, result))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("items", (None, [], ["invalid"], [{"role": "assistant", "content": "none"}], [_USER]))
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
def test_history_requires_older_prefix(items: object, protocol: NativeProtocol) -> None:
|
||||
result: Final = split_native_history({"input" if protocol == "responses" else "messages": items}, protocol)
|
||||
assert isinstance(result, CompactionHistoryError) and result.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("previous_response_id", "conversation"))
|
||||
def test_unresolved_history_rejected(field: str) -> None:
|
||||
result: Final = split_native_history({"input": [_USER, _USER], field: "server-state"}, "responses")
|
||||
assert isinstance(result, CompactionHistoryError) and "Server-side" in result.message
|
||||
|
||||
|
||||
def test_responses_retains_canonical_window_and_only_removes_null_compaction_created_by() -> None:
|
||||
retained: Final = {"type": "message", "role": "user", "content": "retained", "created_by": None}
|
||||
compacted: Final = {"type": "compaction", "encrypted_content": " opaque ", "created_by": None, "other": None}
|
||||
extra: Final = {**compacted, "created_by": "keep"}
|
||||
unknown: Final = {"type": "future_item", "metadata": {"created_by": None}}
|
||||
response: Final = {"output": [retained, compacted, extra, unknown]}
|
||||
original: Final = deepcopy(response)
|
||||
result: Final = native_compaction_result(response, _HISTORY, "responses")
|
||||
assert result == (
|
||||
retained,
|
||||
{"type": "compaction", "encrypted_content": " opaque ", "other": None},
|
||||
extra,
|
||||
unknown,
|
||||
_USER,
|
||||
)
|
||||
assert json.loads(json.dumps(result)) == [dict(item) for item in result]
|
||||
assert response == original
|
||||
|
||||
|
||||
def test_messages_preserves_signed_block_and_tail_without_aliasing() -> None:
|
||||
block: Final = {"type": "compaction", "content": " Summary\n", "signature": "exact-signature", "extra": None}
|
||||
response: Final = {"stop_reason": "compaction", "content": [block]}
|
||||
original: Final = deepcopy(response)
|
||||
result: Final = native_compaction_result(response, _HISTORY, "messages")
|
||||
assert result == ({"role": "assistant", "content": [block]}, _USER)
|
||||
assert json.loads(json.dumps(result)) == [dict(item) for item in result]
|
||||
result[0]["content"][0]["signature"] = "changed-private-copy"
|
||||
assert response == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"protocol,field", (("responses", "encrypted_content"), ("messages", "signature"), ("messages", "content"))
|
||||
)
|
||||
@pytest.mark.parametrize("state", (None, "", " ", 42))
|
||||
def test_invalid_compaction_state_rejected(protocol: NativeProtocol, field: str, state: object) -> None:
|
||||
valid: Final = {"type": "compaction", "content": "summary", "signature": "signed", "encrypted_content": "opaque"}
|
||||
block: Final = {**valid, field: state}
|
||||
response: Final = {"stop_reason": "compaction", "content": [block], "output": [block]}
|
||||
assert isinstance(native_compaction_result(response, _HISTORY, protocol), CompactionHistoryError)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocks", (None, [], ["invalid"], [{"type": "text", "text": "plain summary"}]))
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
def test_missing_native_output_rejected(blocks: object, protocol: NativeProtocol) -> None:
|
||||
response: Final = {"stop_reason": "compaction", "content": blocks, "output": blocks}
|
||||
assert isinstance(native_compaction_result(response, _HISTORY, protocol), CompactionHistoryError)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("stop_reason", "count"), (("end_turn", 1), ("compaction", 2)))
|
||||
def test_messages_requires_one_block_and_compaction_stop(stop_reason: str, count: int) -> None:
|
||||
block: Final = {"type": "compaction", "content": "summary", "signature": "signed"}
|
||||
response: Final = {"stop_reason": stop_reason, "content": [block] * count}
|
||||
assert isinstance(native_compaction_result(response, _HISTORY, "messages"), CompactionHistoryError)
|
||||
|
||||
|
||||
def test_beta_headers_merge_case_insensitively_and_idempotently() -> None:
|
||||
required: Final = compaction_headers(None)["anthropic-beta"]
|
||||
existing: Final = {"Anthropic-Beta": f"other, {required}", "anthropic-beta": "other,third", "x-test": "keep"}
|
||||
original: Final = dict(existing)
|
||||
merged: Final = compaction_headers(existing)
|
||||
assert merged == {"anthropic-beta": f"other,{required},third", "x-test": "keep"}
|
||||
assert compaction_headers(merged) == merged and existing == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("headers", ([], {"x-test": 1}, {"x-test": b"bytes"}, {"anthropic-beta": None}))
|
||||
def test_invalid_headers_fail_at_payload_boundary(headers: object) -> None:
|
||||
with pytest.raises(ValueError, match="validation error"):
|
||||
native_compaction_payload(_HISTORY, {"extra_headers": headers}, "messages")
|
||||
|
|
@ -328,8 +328,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
"required": ["city"],
|
||||
}
|
||||
|
||||
def test_transform_decodes_json_string_tool_parameters_on_compact_request(self):
|
||||
"""The compact request path builds the same wire body, so it must decode too."""
|
||||
def test_transform_compact_omits_json_string_tool_parameters(self):
|
||||
_url, data = self.config.transform_compact_response_api_request(
|
||||
model=self.model,
|
||||
input="weather in Paris",
|
||||
|
|
@ -341,7 +340,7 @@ class TestOpenAIResponsesAPIConfig:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert data["tools"][0]["parameters"] == {"type": "object"}
|
||||
assert "tools" not in data
|
||||
|
||||
@pytest.mark.parametrize("raw_parameters", ['"just a string"', "not json at all", "[1, 2, 3]", 42])
|
||||
def test_transform_rejects_tool_parameters_that_are_not_an_object(self, raw_parameters: object):
|
||||
|
|
@ -386,13 +385,12 @@ class TestOpenAIResponsesAPIConfig:
|
|||
assert "parameters" not in result["tools"][2]
|
||||
assert result["tools"][3] == {"type": "web_search_preview"}
|
||||
|
||||
def test_transform_compact_drops_foreign_tool_call_item_ids(self):
|
||||
"""The compact request path replays input the same way, so it must
|
||||
apply the same id drop."""
|
||||
@pytest.mark.parametrize("item_id, retained", [("toolu_01Foreign", False), ("fc_native", True)])
|
||||
def test_transform_compact_sanitizes_tool_call_item_ids(self, item_id: str, retained: bool) -> None:
|
||||
replayed_input = [
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "toolu_01Foreign",
|
||||
"id": item_id,
|
||||
"call_id": "toolu_01Foreign",
|
||||
"name": "get_weather",
|
||||
"arguments": "{}",
|
||||
|
|
@ -408,7 +406,9 @@ class TestOpenAIResponsesAPIConfig:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert "id" not in data["input"][0]
|
||||
assert ("id" in data["input"][0]) is retained
|
||||
if retained:
|
||||
assert data["input"][0]["id"] == item_id
|
||||
assert data["input"][0]["call_id"] == "toolu_01Foreign"
|
||||
|
||||
def test_transform_streaming_response(self):
|
||||
|
|
@ -1915,7 +1915,7 @@ class TestFlattenToolSchemaCombinatorsWiring:
|
|||
assert set(nested_parameters["properties"]) == {"id", "enabled", "schedule"}
|
||||
assert json.loads(json.dumps(result["tools"])) == result["tools"]
|
||||
|
||||
def test_openai_compact_request_flattens_top_level_anyof(self):
|
||||
def test_openai_compact_request_omits_anyof_tools(self):
|
||||
_, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request(
|
||||
model="gpt-4o",
|
||||
input="hi",
|
||||
|
|
@ -1925,7 +1925,7 @@ class TestFlattenToolSchemaCombinatorsWiring:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert "anyOf" not in data["tools"][0]["parameters"]
|
||||
assert "tools" not in data
|
||||
|
||||
def test_openai_leaves_tools_without_rejected_keys_alone(self):
|
||||
clean_tool = {
|
||||
|
|
@ -2077,7 +2077,7 @@ class TestToolSchemaRegexPatternWiring:
|
|||
|
||||
assert result["tools"][0]["tools"][0]["parameters"]["properties"]["field"] == {"type": "string"}
|
||||
|
||||
def test_openai_compact_request_drops_patterns(self):
|
||||
def test_openai_compact_request_omits_pattern_tools(self):
|
||||
_, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request(
|
||||
model="gpt-5.6",
|
||||
input="hi",
|
||||
|
|
@ -2087,7 +2087,7 @@ class TestToolSchemaRegexPatternWiring:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert data["tools"][0]["parameters"]["properties"]["field"] == {"type": "string"}
|
||||
assert "tools" not in data
|
||||
|
||||
def test_non_openai_subclass_keeps_patterns(self):
|
||||
from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig
|
||||
|
|
@ -2208,3 +2208,34 @@ class TestReasoningFollowsModelSupport:
|
|||
drop_params=True,
|
||||
)
|
||||
assert mapped["reasoning"] == {"effort": "medium"}
|
||||
|
||||
|
||||
def test_native_compact_request_allowlist_preserves_input_and_caller_params() -> None:
|
||||
from copy import deepcopy
|
||||
from openai.types.responses import ResponseInputParam
|
||||
|
||||
model: Final = "native-compact-test"
|
||||
native_input: Final[ResponseInputParam] = [
|
||||
{"id": "cmp_test", "type": "compaction", "encrypted_content": "opaque"},
|
||||
{"role": "user", "content": "latest question"},
|
||||
]
|
||||
allowed: Final = {
|
||||
"instructions": "keep these rules", "previous_response_id": "resp_previous",
|
||||
"prompt_cache_key": "cache-prefix", "prompt_cache_retention": "24h",
|
||||
}
|
||||
params: Final[dict[str, object]] = {
|
||||
**allowed, "metadata": {"trace": "caller-owned"}, "stream": True,
|
||||
"tools": [{"type": "function", "name": "lookup", "parameters": '{"type":"object"}'}],
|
||||
"reasoning": {"effort": "medium"}, "max_output_tokens": 128,
|
||||
}
|
||||
original: Final = deepcopy((native_input, params))
|
||||
_, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request(
|
||||
model=model,
|
||||
input=native_input,
|
||||
response_api_optional_request_params=params,
|
||||
api_base="https://api.openai.com/v1/responses",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert data == {"model": model, "input": original[0], **allowed}
|
||||
assert (native_input, params) == original
|
||||
|
|
|
|||
233
tests/test_litellm/proxy/common_utils/test_context_compaction.py
Normal file
233
tests/test_litellm/proxy/common_utils/test_context_compaction.py
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
import asyncio
|
||||
from collections.abc import Generator, Mapping
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import FastAPI, Request
|
||||
from pydantic import TypeAdapter
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.context_compaction import (
|
||||
_SummaryRequest,
|
||||
_summary_request,
|
||||
proxy_summary_executor_scope,
|
||||
summary_request_updates,
|
||||
)
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
_PROXY_MaxParallelRequestsHandler_v3,
|
||||
_request_stash,
|
||||
get_request_stash,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache, hash_token
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
NativeRequest,
|
||||
current_summary_executor,
|
||||
)
|
||||
|
||||
_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_KEY: Final = "sk-test-context-compaction"
|
||||
_KEY_HASH: Final = hash_token(_KEY)
|
||||
_MESSAGES: Final = [{"role": "user", "content": "Private older conversation"}]
|
||||
|
||||
|
||||
def parent_request(app: ASGIApp) -> Request:
|
||||
return Request({
|
||||
"type": "http", "method": "POST", "scheme": "https", "path": "/gateway/v1/messages",
|
||||
"root_path": "/gateway", "query_string": b"", "server": ("gateway.test", 443),
|
||||
"client": ("192.0.2.10", 2345), "app": app,
|
||||
"headers": [(b"authorization", f"Bearer {_KEY}".encode()), (b"content-length", b"99999"),
|
||||
(b"x-litellm-call-id", b"parent"), (b"litellm-disable-message-redaction", b"true"),
|
||||
(b"x-app", b"cli"), (b"x-claude-code-agent-id", b"agent"),
|
||||
(b"x-claude-code-session-id", b"session"), (b"x-litellm-model-id", b"target-id"),
|
||||
(b"x-litellm-model", b"target"), (b"anthropic-beta", b"existing-beta")],
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def summary_proxy(
|
||||
protocol: Literal["responses", "messages"], monkeypatch: pytest.MonkeyPatch,
|
||||
) -> Generator[tuple[_PROXY_MaxParallelRequestsHandler_v3, AsyncMock], None, None]:
|
||||
limiter: Final = _PROXY_MaxParallelRequestsHandler_v3(InternalUsageCache(DualCache()))
|
||||
inference: Final = AsyncMock()
|
||||
monkeypatch.setattr(litellm, "acompact_responses" if protocol == "responses" else "anthropic_messages", inference)
|
||||
model: Final = "openai/gpt-5.6" if protocol == "responses" else "anthropic/claude-sonnet-5"
|
||||
router: Final = litellm.Router(model_list=[{
|
||||
"model_name": "summary", "litellm_params": {"model": model, "api_key": "test-only-key"},
|
||||
}])
|
||||
monkeypatch.setattr(proxy_server, "master_key", "sk-test-master")
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache())
|
||||
monkeypatch.setattr(proxy_server.proxy_logging_obj, "max_parallel_request_limiter", limiter)
|
||||
monkeypatch.setattr(litellm, "callbacks", [limiter])
|
||||
token: Final = _request_stash.set(None)
|
||||
try:
|
||||
yield limiter, inference
|
||||
finally:
|
||||
_request_stash.reset(token)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("outcome", ["success", "denied", "route-denied", "cancel", "timeout", "failed"])
|
||||
@pytest.mark.parametrize("protocol", ["responses", "messages"])
|
||||
async def test_real_proxy_summary_auth_identity_and_lifecycle(
|
||||
outcome: Literal["success", "denied", "route-denied", "cancel", "timeout", "failed"], protocol: Literal["responses", "messages"],
|
||||
summary_proxy: tuple[_PROXY_MaxParallelRequestsHandler_v3, AsyncMock],
|
||||
) -> None:
|
||||
limiter, inference = summary_proxy
|
||||
native_payload: Final = (
|
||||
{"input": _MESSAGES, "instructions": "Preserve the constraints"} if protocol == "responses" else
|
||||
{"messages": _MESSAGES, "max_tokens": 128, "system": "Preserve the constraints", "tools": [],
|
||||
"compaction": {"type": "summarize"}}
|
||||
)
|
||||
native: Final = NativeRequest(protocol, {
|
||||
**native_payload, "model": "target", "api_key": "payload-credential", "extra_body": {"model": "target"},
|
||||
"stream": True,
|
||||
"extra_headers": {
|
||||
"Anthropic-Beta": "existing-beta, compact-2026-09-04",
|
||||
"authorization": "Bearer other", "x-api-key": "other",
|
||||
},
|
||||
}, frozenset({"native-test"}))
|
||||
result: Final = {
|
||||
"id": "native-result", "usage": {"input_tokens": 10, "output_tokens": 2},
|
||||
**({"object": "response.compaction", "output": [{"type": "compaction", "encrypted_content": "opaque"}]}
|
||||
if protocol == "responses" else {
|
||||
"type": "message", "role": "assistant", "stop_reason": "compaction",
|
||||
"content": [{"type": "compaction", "content": "opaque", "signature": "signed"}],
|
||||
}),
|
||||
}
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key=_KEY_HASH, models=["target"] if outcome == "denied" else ["target", "summary"],
|
||||
max_parallel_requests=1, rpm_limit=10,
|
||||
allowed_routes=["/v1/chat/completions"] if outcome == "route-denied" else ["llm_api_routes"],
|
||||
)
|
||||
await proxy_server.user_api_key_cache.async_set_cache(_KEY_HASH, auth, model_type=UserAPIKeyAuth)
|
||||
parent_data: Final = {
|
||||
"model": "target", "litellm_call_id": "parent", "turn_off_message_logging": True,
|
||||
"allowed_model_region": "eu", "user": "end-user", "litellm_session_id": "session",
|
||||
"metadata": {"tags": ["cost-center"]},
|
||||
}
|
||||
await limiter.async_pre_call_hook(auth, DualCache(), parent_data, "acompletion")
|
||||
parent: Final = get_request_stash()
|
||||
assert parent is not None and parent.parallel_slot is not None
|
||||
acquisition: Final = parent.parallel_slot
|
||||
entered: Final = asyncio.Event()
|
||||
stopped: Final = asyncio.Event()
|
||||
application: Final = AsyncMock(wraps=proxy_server.app.__call__)
|
||||
|
||||
async def receive(**provider_data: object) -> Mapping[str, object]:
|
||||
data: Final = _BODY.validate_python(_BODY.validate_python(provider_data["proxy_server_request"])["body"])
|
||||
assert get_request_stash() is not parent
|
||||
assert parent.parallel_slot is acquisition
|
||||
child: Final = Request(application.call_args.args[0])
|
||||
assert child.url.path == "/gateway/v1/" + ("responses/compact" if protocol == "responses" else "messages")
|
||||
assert child.headers["authorization"] == f"Bearer {_KEY}"
|
||||
assert "x-api-key" not in child.headers
|
||||
assert child.headers["anthropic-beta"] == "existing-beta,compact-2026-09-04"
|
||||
if protocol == "responses":
|
||||
assert "stream" not in provider_data
|
||||
else:
|
||||
assert provider_data["stream"] is False
|
||||
assert child.client.host == "192.0.2.10"
|
||||
assert not {"litellm-disable-message-redaction", "x-app", "x-claude-code-agent-id",
|
||||
"x-litellm-model-id", "x-litellm-model"}.intersection(child.headers)
|
||||
assert child.headers["x-claude-code-session-id"] == "session"
|
||||
assert data["litellm_call_id"] != "parent"
|
||||
for name, expected in (
|
||||
("turn_off_message_logging", True), ("num_retries", 0), ("max_retries", 0),
|
||||
("disable_fallbacks", True), ("allowed_model_region", "eu"), ("user", "end-user"),
|
||||
("litellm_session_id", "session"), ("model", "summary"), *native_payload.items(),
|
||||
):
|
||||
assert data[name] == expected
|
||||
assert provider_data.get("api_key") != "payload-credential"
|
||||
assert "extra_body" not in data
|
||||
metadata: Final = _BODY.validate_python(data["litellm_metadata"])
|
||||
assert metadata[INTERNAL_CALL_ORIGIN_METADATA_KEY] == "context_compaction"
|
||||
assert metadata["user_api_key"] == _KEY_HASH
|
||||
assert "cost-center" in metadata["tags"]
|
||||
with pytest.raises(ValueError, match="nested"):
|
||||
await executor("summary", native, 10)
|
||||
entered.set()
|
||||
try:
|
||||
if outcome in ("cancel", "timeout"):
|
||||
await asyncio.Event().wait()
|
||||
if outcome == "failed":
|
||||
raise litellm.APIError(503, "Private older conversation", llm_provider="test", model="summary")
|
||||
return result
|
||||
finally:
|
||||
stopped.set()
|
||||
|
||||
inference.side_effect = receive
|
||||
assert current_summary_executor() is None
|
||||
with proxy_summary_executor_scope(parent_request(application), parent_data, limiter):
|
||||
executor: Final = current_summary_executor()
|
||||
assert executor is not None
|
||||
task: Final = asyncio.create_task(executor("summary", native, 0.2 if outcome == "timeout" else 10))
|
||||
if outcome in ("denied", "route-denied", "failed"):
|
||||
with pytest.raises(litellm.APIError) as denied:
|
||||
await task
|
||||
assert denied.value.status_code in ((503,) if outcome == "failed" else (401, 403))
|
||||
assert "Private older conversation" not in str(denied.value)
|
||||
assert inference.await_count == int(outcome == "failed")
|
||||
cached: Final = await proxy_server.user_api_key_cache.async_get_cache(_KEY_HASH, model_type=UserAPIKeyAuth)
|
||||
assert cached.models == auth.models and cached.spend == 0
|
||||
else:
|
||||
if outcome == "success":
|
||||
assert await task == result
|
||||
else:
|
||||
await asyncio.wait_for(entered.wait(), 5)
|
||||
if outcome == "cancel":
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError if outcome == "cancel" else TimeoutError):
|
||||
await task
|
||||
inference.assert_awaited_once()
|
||||
assert stopped.is_set()
|
||||
assert current_summary_executor() is None
|
||||
assert get_request_stash() is parent and parent.parallel_slot is acquisition
|
||||
assert not parent.reservation_released
|
||||
with pytest.raises(ValueError, match="closed"):
|
||||
await executor("summary", native, 10)
|
||||
await limiter.async_post_call_success_hook(parent_data, auth, ModelResponse())
|
||||
assert parent.parallel_slot is None
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=proxy_server.app)) as client:
|
||||
rejected: Final = await client.post(
|
||||
"https://gateway.test/v1/" + ("responses/compact" if protocol == "responses" else "messages"),
|
||||
headers={"authorization": f"Bearer {_KEY}"},
|
||||
json={**native_payload, "model": "summary", "turn_off_message_logging": True},
|
||||
)
|
||||
assert rejected.is_client_error and "turn_off_message_logging" in rejected.text
|
||||
assert proxy_server.general_settings == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parent_redact", [False, True, None])
|
||||
@pytest.mark.parametrize("recipient_redact", [False, True, None])
|
||||
@pytest.mark.parametrize("global_redact", [False, True])
|
||||
def test_summary_privacy_never_lowers_recipient_policy(
|
||||
parent_redact: bool | None, recipient_redact: bool | None,
|
||||
global_redact: bool, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", global_redact)
|
||||
summary: Final = _SummaryRequest(call_id="parent", redact=parent_redact)
|
||||
token: Final = _summary_request.set(summary)
|
||||
recipient_data: Final = {"turn_off_message_logging": recipient_redact, "metadata": {}}
|
||||
try:
|
||||
updates: Final = summary_request_updates(
|
||||
parent_request(FastAPI()), UserAPIKeyAuth(), recipient_data, "metadata"
|
||||
)
|
||||
if parent_redact is True or recipient_redact is True or global_redact:
|
||||
assert updates["turn_off_message_logging"] is True
|
||||
assert summary.redact is True
|
||||
else:
|
||||
assert "turn_off_message_logging" not in updates
|
||||
finally:
|
||||
_summary_request.reset(token)
|
||||
|
|
@ -10,7 +10,7 @@ import time
|
|||
from collections.abc import Sequence
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, Final, List, Optional
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -27,6 +27,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
|||
_request_stash,
|
||||
get_or_create_request_stash,
|
||||
get_request_stash,
|
||||
isolated_summary_request_stash,
|
||||
)
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
_PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler,
|
||||
|
|
@ -6066,6 +6067,104 @@ def test_internal_call_origin_success_ops_are_skipped():
|
|||
|
||||
assert charged
|
||||
assert skipped == []
|
||||
summary = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: "context_compaction"}),
|
||||
response_obj=response,
|
||||
rate_limit_type="output",
|
||||
)
|
||||
assert summary == charged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("parent_owner", ["parent", "unrelated", None])
|
||||
async def test_summary_borrows_only_live_direct_parent_and_preserves_rpm(parent_owner):
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(cache))
|
||||
auth = UserAPIKeyAuth(api_key=hash_token("sk-summary"), max_parallel_requests=1, rpm_limit=2)
|
||||
parent_data = {"model": "target", "litellm_call_id": "parent"}
|
||||
await handler.async_pre_call_hook(auth, cache, parent_data, "acompletion")
|
||||
parent = get_request_stash()
|
||||
acquisition = parent.parallel_slot
|
||||
counter_key = acquisition["counter_keys"][0]
|
||||
child_data = {"model": "summary", "litellm_call_id": "child"}
|
||||
async with isolated_summary_request_stash(parent, parent_owner, "child") as child:
|
||||
assert child is not parent
|
||||
if parent_owner == "parent":
|
||||
await handler.async_pre_call_hook(auth, cache, child_data, "acompletion")
|
||||
assert child.parallel_slot is None
|
||||
await handler.async_post_call_success_hook(child_data, auth, ModelResponse())
|
||||
with pytest.raises(HTTPException):
|
||||
async with isolated_summary_request_stash(child, "child", "grandchild"):
|
||||
await handler.async_pre_call_hook(
|
||||
auth, cache, {"model": "summary", "litellm_call_id": "grandchild"}, "acompletion"
|
||||
)
|
||||
else:
|
||||
with pytest.raises(HTTPException):
|
||||
await handler.async_pre_call_hook(auth, cache, child_data, "acompletion")
|
||||
assert parent.parallel_slot is acquisition
|
||||
assert handler._gauge_in_flight_from_cache_value(await cache.async_get_cache(key=counter_key)) == 1
|
||||
assert get_request_stash() is parent
|
||||
await handler.async_post_call_success_hook(parent_data, auth, ModelResponse())
|
||||
assert handler._gauge_in_flight_from_cache_value(await cache.async_get_cache(key=counter_key)) == 0
|
||||
with pytest.raises(HTTPException):
|
||||
await handler.async_pre_call_hook(auth, cache, {"model": "summary"}, "acompletion")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summary_lifetime_keeps_parent_release_locked():
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(cache))
|
||||
auth = UserAPIKeyAuth(api_key=hash_token("sk-summary-lock"), max_parallel_requests=1)
|
||||
data = {"model": "target", "litellm_call_id": "parent"}
|
||||
await handler.async_pre_call_hook(auth, cache, data, "acompletion")
|
||||
parent = get_request_stash()
|
||||
async with isolated_summary_request_stash(parent, "parent", "child"):
|
||||
release = asyncio.create_task(handler._release_stashed_parallel_slot(parent, None))
|
||||
await asyncio.sleep(0)
|
||||
assert not release.done()
|
||||
assert parent.parallel_slot is not None
|
||||
await release
|
||||
assert parent.parallel_slot is None
|
||||
|
||||
|
||||
def test_summary_gauge_exclusion_retains_new_model_gauge_and_windowed_limits():
|
||||
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(DualCache()))
|
||||
descriptors = (
|
||||
{"key": "api_key", "value": "caller", "rate_limit": {
|
||||
"max_parallel_requests": 1, "requests_per_unit": 5, "tokens_per_unit": 100,
|
||||
}},
|
||||
{"key": "model_per_key", "value": "summary:caller", "rate_limit": {"max_parallel_requests": 1}},
|
||||
)
|
||||
all_keys, all_metadata, all_gauges = handler._collect_windowed_keys_and_gauges(descriptors, False)
|
||||
keys, metadata, gauges = handler._collect_windowed_keys_and_gauges(
|
||||
descriptors, False, frozenset({all_gauges[0]["counter_key"]})
|
||||
)
|
||||
assert keys == all_keys
|
||||
assert metadata == all_metadata
|
||||
assert gauges == [all_gauges[1]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summary_origin_reconciles_actual_key_and_team_tpm():
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(InternalUsageCache(cache))
|
||||
key = hash_token("sk-summary-accounting")
|
||||
stash = get_or_create_request_stash()
|
||||
stash.owner_litellm_call_id = "summary-child"
|
||||
stash.reserved_tokens = 40
|
||||
stash.reserved_scopes = frozenset({("api_key", key), ("team", "team")})
|
||||
for scope, value in stash.reserved_scopes:
|
||||
await cache.async_set_cache(handler.create_rate_limit_keys(scope, value, "tokens"), 40, ttl=60)
|
||||
response = ModelResponse(usage=Usage(prompt_tokens=7, completion_tokens=5, total_tokens=12))
|
||||
await handler.async_log_success_event(
|
||||
kwargs={
|
||||
"litellm_call_id": "summary-child", "model": "summary",
|
||||
"standard_logging_object": {"metadata": {"user_api_key_hash": key, "user_api_key_team_id": "team"}},
|
||||
"litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: "context_compaction"}},
|
||||
}, response_obj=response, start_time=None, end_time=None,
|
||||
)
|
||||
for scope, value in stash.reserved_scopes:
|
||||
assert await cache.async_get_cache(handler.create_rate_limit_keys(scope, value, "tokens")) == 12
|
||||
|
||||
|
||||
def _conflicting_budget_bodies() -> Dict[str, Dict[str, object]]:
|
||||
|
|
@ -6609,6 +6708,58 @@ class _DelayedCapacityUsageCache:
|
|||
await self.finish_release.wait()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summary_admission_cancellation_releases_only_child_slots_despite_repeated_cancel():
|
||||
cache: Final = _DelayedCapacityUsageCache()
|
||||
parent_handler: Final = _PROXY_MaxParallelRequestsHandler(cache.delegate)
|
||||
child_handler: Final = _PROXY_MaxParallelRequestsHandler(cache)
|
||||
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-summary-cancel-admission"), max_parallel_requests=1)
|
||||
parent_data: Final = {"model": "target", "litellm_call_id": "parent"}
|
||||
await parent_handler.async_pre_call_hook(auth, cache.dual_cache, parent_data, "acompletion")
|
||||
parent: Final = get_request_stash()
|
||||
assert parent is not None and parent.parallel_slot is not None
|
||||
parent_acquisition: Final = parent.parallel_slot
|
||||
parent_key: Final = parent_acquisition["counter_keys"][0]
|
||||
child_key: Final = child_handler.create_rate_limit_keys("model_per_key", "summary:caller", "max_parallel_requests")
|
||||
descriptors: Final = (
|
||||
{"key": "api_key", "value": auth.api_key, "rate_limit": {"max_parallel_requests": 1}},
|
||||
{"key": "model_per_key", "value": "summary:caller", "rate_limit": {"max_parallel_requests": 1}},
|
||||
)
|
||||
|
||||
async def admit_child() -> None:
|
||||
async with isolated_summary_request_stash(parent, "parent", "child"):
|
||||
await child_handler._admit_request_capacity(
|
||||
descriptors, auth, "child-slot", (child_key,), frozenset(parent_acquisition["counter_keys"])
|
||||
)
|
||||
pytest.fail("cancelled summary admission entered provider body")
|
||||
|
||||
task: Final = asyncio.create_task(admit_child())
|
||||
try:
|
||||
await asyncio.wait_for(cache.acquired.wait(), timeout=2)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
cache.finish_admission.set()
|
||||
await asyncio.wait_for(cache.releasing.wait(), timeout=2)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
assert not task.done()
|
||||
cache.finish_release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(task, timeout=2)
|
||||
assert get_request_stash() is parent
|
||||
assert parent.parallel_slot is parent_acquisition
|
||||
assert parent_handler._gauge_in_flight_from_cache_value(await cache.dual_cache.async_get_cache(parent_key)) == 1
|
||||
assert child_handler._gauge_in_flight_from_cache_value(await cache.dual_cache.async_get_cache(child_key)) == 0
|
||||
finally:
|
||||
cache.finish_admission.set()
|
||||
cache.finish_release.set()
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
await parent_handler.async_post_call_success_hook(parent_data, auth, ModelResponse())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_capacity_finishes_admission_and_release_despite_repeated_cancel():
|
||||
cache = _DelayedCapacityUsageCache()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -10,6 +10,7 @@ from litellm.proxy._types import (
|
|||
LiteLLM_TeamTable,
|
||||
LitellmUserRoles,
|
||||
Member,
|
||||
ProxyException,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.management_helpers.auto_router_permissions import (
|
||||
|
|
@ -120,6 +121,29 @@ def test_all_tier_parameter_representations_reject_privileged_overrides(
|
|||
assert denied.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("denied_scope", [None, "team", "key"])
|
||||
async def test_compaction_recipient_requires_normal_model_access(
|
||||
catalog: Router, denied_scope: Literal["team", "key"] | None
|
||||
) -> None:
|
||||
operation: Final = authorize_member_auto_router_dependencies(
|
||||
config=validate_member_auto_router_config(
|
||||
{"tiers": {"SIMPLE": "allowed"}, "context_window_compaction_model": "other"}
|
||||
),
|
||||
default_model=None,
|
||||
user_api_key_dict=_actor(models=["allowed"] if denied_scope == "key" else ["allowed", "other"]),
|
||||
team=_team(models=["allowed"] if denied_scope == "team" else ["allowed", "other"]),
|
||||
prisma_client=_Client(),
|
||||
llm_router=catalog,
|
||||
)
|
||||
if denied_scope is None:
|
||||
await operation
|
||||
return
|
||||
with pytest.raises(ProxyException, match="other") as denied:
|
||||
await operation
|
||||
assert str(denied.value.code) == "403"
|
||||
|
||||
|
||||
def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> None:
|
||||
validated: Final = validate_member_auto_router_config(
|
||||
{"tiers": {"SIMPLE": [{"model_name": "allowed", "litellm_params": {"reasoning_effort": "low"}}]}}
|
||||
|
|
@ -183,7 +207,10 @@ async def test_member_updates_restrict_fields_and_preserve_an_inherited_default(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("target", ["missing", "nested"])
|
||||
async def test_member_dependencies_require_plain_configured_models(target: str) -> None:
|
||||
@pytest.mark.parametrize("role", ["tier", "compaction"])
|
||||
async def test_member_dependencies_require_plain_configured_models(
|
||||
target: str, role: Literal["tier", "compaction"]
|
||||
) -> None:
|
||||
catalog: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "allowed", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}},
|
||||
|
|
@ -198,10 +225,14 @@ async def test_member_dependencies_require_plain_configured_models(target: str)
|
|||
)
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
await authorize_member_auto_router_dependencies(
|
||||
config=validate_member_auto_router_config({"tiers": {"SIMPLE": target}}),
|
||||
config=validate_member_auto_router_config(
|
||||
{"tiers": {"SIMPLE": target}}
|
||||
if role == "tier"
|
||||
else {"tiers": {"SIMPLE": "allowed"}, "context_window_compaction_model": target}
|
||||
),
|
||||
default_model=None,
|
||||
user_api_key_dict=_actor(models=[target]),
|
||||
team=_team(models=[target]),
|
||||
user_api_key_dict=_actor(models=["allowed", target]),
|
||||
team=_team(models=["allowed", target]),
|
||||
prisma_client=_Client(),
|
||||
llm_router=catalog,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,273 @@
|
|||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from contextvars import Context, copy_context
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field, replace
|
||||
from queue import SimpleQueue
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.compaction import NativeProtocol
|
||||
from litellm.router_strategy.complexity_router import context_compaction as core
|
||||
|
||||
pytestmark: Final = pytest.mark.asyncio
|
||||
|
||||
|
||||
def _budget(protocol: NativeProtocol, identifier: str = "target") -> core.ModelBudget:
|
||||
provider: Final = "openai" if protocol == "responses" else "anthropic"
|
||||
return core.ModelBudget(f"{provider}/native-test", 2_000, 256, deployment_id=identifier)
|
||||
|
||||
|
||||
_HISTORY: Final = (
|
||||
{"role": "user", "content": "old"},
|
||||
{"role": "assistant", "content": "answer"},
|
||||
{"role": "user", "content": "latest"},
|
||||
)
|
||||
|
||||
|
||||
def _payload(protocol: NativeProtocol) -> Mapping[str, object]:
|
||||
return {
|
||||
"input" if protocol == "responses" else "messages": list(_HISTORY),
|
||||
"instructions" if protocol == "responses" else "system": "rules",
|
||||
"tools": [],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Rig:
|
||||
counts: tuple[int | Exception, ...] = (2_000, 100)
|
||||
local_count: int = 2_000
|
||||
error: Exception | None = None
|
||||
block: bool = False
|
||||
calls: SimpleQueue[tuple[str, core.NativeRequest, Context]] = field(default_factory=SimpleQueue)
|
||||
native: SimpleQueue[tuple[core.ModelBudget, Mapping[str, object], NativeProtocol]] = field(
|
||||
default_factory=SimpleQueue
|
||||
)
|
||||
local: SimpleQueue[str] = field(default_factory=SimpleQueue)
|
||||
started: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
async def counter(self, model: str, payload: Mapping[str, object]) -> int:
|
||||
self.local.put(model)
|
||||
return self.local_count
|
||||
|
||||
async def native_counter(
|
||||
self, target: core.ModelBudget, payload: Mapping[str, object], protocol: NativeProtocol, timeout: float
|
||||
) -> int:
|
||||
assert timeout > 0
|
||||
self.native.put((target, payload, protocol))
|
||||
result: Final = self.counts[(self.native.qsize() - 1) % len(self.counts)]
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
|
||||
async def execute(self, model: str, request: core.NativeRequest, timeout: float) -> Mapping[str, object]:
|
||||
assert core.current_native_request() is request and timeout > 0
|
||||
self.calls.put((model, request, copy_context()))
|
||||
self.started.set()
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
if self.block:
|
||||
await asyncio.Event().wait()
|
||||
if request.protocol == "responses":
|
||||
return {"output": [{"type": "compaction", "encrypted_content": "opaque"}]}
|
||||
return {
|
||||
"stop_reason": "compaction",
|
||||
"content": [{"type": "compaction", "content": "native context", "signature": "signed"}],
|
||||
}
|
||||
|
||||
|
||||
async def _run(
|
||||
rig: _Rig,
|
||||
protocol: NativeProtocol = "responses",
|
||||
state: core.CompactionState | None = None,
|
||||
payload: Mapping[str, object] | None = None,
|
||||
budgets: tuple[core.ModelBudget, ...] | None = None,
|
||||
target: core.ModelBudget | None = None,
|
||||
route: Literal["chat", "responses", "messages"] | None = None,
|
||||
) -> core.CompactedRequest | core.CompactionFailure | None:
|
||||
active: Final = core.CompactionState() if state is None else state
|
||||
active.arm(active.model or "compactors")
|
||||
recipient: Final = _budget(protocol) if target is None else target
|
||||
candidates: Final = (
|
||||
(replace(recipient, input_limit=10_000, output_limit=core.SUMMARY_OUTPUT_TOKENS, deployment_id="allowed"),)
|
||||
if budgets is None
|
||||
else budgets
|
||||
)
|
||||
return await core.prepare_compaction(
|
||||
_payload(protocol) if payload is None else payload,
|
||||
recipient,
|
||||
active,
|
||||
candidates,
|
||||
rig.execute,
|
||||
rig.counter,
|
||||
route or protocol,
|
||||
rig.native_counter,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
@pytest.mark.parametrize("mode", ("success", "error", "timeout", "cancel", "expired"))
|
||||
async def test_candidates_recipient_and_context_cleanup(protocol: NativeProtocol, mode: str) -> None:
|
||||
target: Final = _budget(protocol)
|
||||
good: Final = replace(target, input_limit=10_000, output_limit=core.SUMMARY_OUTPUT_TOKENS, deployment_id="good")
|
||||
bad: Final = (
|
||||
replace(good, deployment_id="small", input_limit=1_000),
|
||||
replace(good, deployment_id="output", output_limit=1),
|
||||
replace(good, deployment_id=""),
|
||||
replace(good, deployment_id="other", model="anthropic/test" if protocol == "responses" else "openai/test"),
|
||||
)
|
||||
rig: Final = _Rig(
|
||||
error=RuntimeError("private native response") if mode == "error" else None, block=mode in ("timeout", "cancel")
|
||||
)
|
||||
state: Final = core.CompactionState(timeout=0 if mode == "expired" else 0.02 if mode == "timeout" else 10)
|
||||
payload: Final = _payload(protocol)
|
||||
original: Final = deepcopy(payload)
|
||||
with core.use_summary_executor(rig.execute):
|
||||
assert core.current_summary_executor() == rig.execute
|
||||
task: Final = asyncio.create_task(_run(rig, protocol, state, payload, (*bad, good), target))
|
||||
if mode == "cancel":
|
||||
await rig.started.wait()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
else:
|
||||
assert isinstance(await task, core.CompactedRequest if mode == "success" else core.CompactionFailure)
|
||||
assert core.current_summary_executor() is None and core.current_native_request() is None and payload == original
|
||||
assert rig.calls.qsize() == (0 if mode == "expired" else 1)
|
||||
if mode != "expired":
|
||||
model, request, inherited = rig.calls.get_nowait()
|
||||
assert model == "compactors" and request.allowed_deployment_ids == frozenset({"good"})
|
||||
assert request.closed.is_set() and inherited.run(core.current_native_request) is None
|
||||
if mode == "success":
|
||||
before, after = (rig.native.get_nowait() for _ in range(2))
|
||||
result: Final = task.result()
|
||||
assert isinstance(result, core.CompactedRequest) and after[1][result.field] == result.value
|
||||
assert before[0] is target and after[0] is target and after[2] == protocol
|
||||
denied: Final = _Rig()
|
||||
assert isinstance(await _run(denied, protocol, budgets=bad), core.CompactionFailure) and denied.calls.empty()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ("local_fit", "native_fit", "chat", "provider", "no_prefix"))
|
||||
async def test_fit_and_unsupported_admission(scenario: str) -> None:
|
||||
rig: Final = _Rig(
|
||||
local_count=10 if scenario == "local_fit" else 2_000, counts=(10,) if scenario == "native_fit" else (2_000,)
|
||||
)
|
||||
target: Final = (
|
||||
replace(_budget("responses"), model="anthropic/test") if scenario == "provider" else _budget("responses")
|
||||
)
|
||||
result: Final = await _run(
|
||||
rig,
|
||||
target=target,
|
||||
payload={"input": [{"role": "user", "content": "latest"}]},
|
||||
route="chat" if scenario == "chat" else "responses",
|
||||
budgets=(),
|
||||
)
|
||||
assert (result is None) if scenario.endswith("fit") else isinstance(result, core.CompactionFailure)
|
||||
assert rig.calls.empty()
|
||||
assert rig.native.qsize() == (0 if scenario in ("local_fit", "chat", "provider") else 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("counts", "paid"),
|
||||
(
|
||||
*((((value,), 0)) for value in (-1, True, False, RuntimeError("private native response"))),
|
||||
*((((2_000, value), 1)) for value in (-1, True, False, 2_000, RuntimeError("private native response"))),
|
||||
),
|
||||
)
|
||||
async def test_native_count_failures_do_not_retry_paid_work(counts: tuple[int | Exception, ...], paid: int) -> None:
|
||||
rig: Final = _Rig(counts=counts)
|
||||
state: Final = core.CompactionState()
|
||||
result: Final = await _run(rig, state=state)
|
||||
if not isinstance(result, core.CompactionFailure):
|
||||
pytest.fail("Invalid native count was accepted")
|
||||
assert "private native response" not in result.message
|
||||
assert isinstance(await _run(rig, state=state), core.CompactionFailure)
|
||||
assert rig.calls.qsize() == paid
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change", ("same", "tail", "prefix", "instructions", "tools", "model", "protocol", "recipient")
|
||||
)
|
||||
async def test_memo_identity_and_one_call_per_request(change: str) -> None:
|
||||
rig: Final = _Rig()
|
||||
state: Final = core.CompactionState()
|
||||
assert isinstance(await _run(rig, state=state), core.CompactedRequest)
|
||||
changed: Final = {"role": "user", "content": "changed"}
|
||||
patches: Final = {
|
||||
"tail": {"input": [*_HISTORY[:-1], changed]},
|
||||
"prefix": {"input": [changed, _HISTORY[-1]]},
|
||||
"instructions": {"instructions": "changed"},
|
||||
"tools": {"tools": [{"type": "function", "name": "lookup"}]},
|
||||
}
|
||||
if change == "model":
|
||||
state.arm("different-compactors")
|
||||
protocol: Final = "messages" if change == "protocol" else "responses"
|
||||
target: Final = _budget(protocol, "retry" if change == "recipient" else "target")
|
||||
result: Final = await _run(rig, protocol, state, {**_payload(protocol), **patches.get(change, {})}, target=target)
|
||||
accepted: Final = change in ("same", "tail", "recipient")
|
||||
assert isinstance(result, core.CompactedRequest if accepted else core.CompactionFailure)
|
||||
assert rig.calls.qsize() == 1 and rig.native.qsize() == (4 if accepted else 3)
|
||||
if isinstance(result, core.CompactedRequest):
|
||||
assert result.value[-1] == (changed if change == "tail" else _HISTORY[-1])
|
||||
assert isinstance(await _run(_Rig()), core.CompactedRequest)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
@pytest.mark.parametrize(
|
||||
"change", ("none", "id", "provider", "defaults", "oversized", "negative", "boolean", "allowance")
|
||||
)
|
||||
async def test_exact_selected_compactor_validation(protocol: NativeProtocol, change: str) -> None:
|
||||
rig: Final = _Rig(counts=({"oversized": 100_000, "negative": -1, "boolean": True}.get(change, 100),))
|
||||
target: Final = replace(_budget(protocol), input_limit=10_000, output_limit=core.SUMMARY_OUTPUT_TOKENS)
|
||||
body: Final = {
|
||||
**_payload(protocol),
|
||||
**({"max_tokens": core.SUMMARY_OUTPUT_TOKENS} if protocol == "messages" else {}),
|
||||
}
|
||||
request: Final = core.NativeRequest(protocol, body, frozenset({target.deployment_id}))
|
||||
selected: Final = replace(
|
||||
target,
|
||||
deployment_id="outsider" if change == "id" else target.deployment_id,
|
||||
model=("anthropic/test" if protocol == "responses" else "openai/test")
|
||||
if change == "provider"
|
||||
else target.model,
|
||||
)
|
||||
overrides: Final = {"defaults": {"instructions": "overridden"}, "allowance": {"max_tokens": 1}}
|
||||
payload: Final = {**body, **overrides.get(change, {})}
|
||||
result: Final = await core.validate_native_recipient(request, selected, payload, 10, rig.native_counter)
|
||||
accepted: Final = change == "none" or (change == "allowance" and protocol == "responses")
|
||||
assert (result is None) if accepted else isinstance(result, core.CompactionFailure)
|
||||
assert rig.native.qsize() == (1 if accepted or change in ("oversized", "negative", "boolean") else 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("patch", "reserve"),
|
||||
(
|
||||
*(
|
||||
({key: value}, reserve)
|
||||
for key in ("max_tokens", "max_completion_tokens", "max_output_tokens")
|
||||
for value, reserve in ((None, 256), (64, 64), (0, None), (True, None), ("64", None), (257, None))
|
||||
),
|
||||
(dict[str, object](), 256),
|
||||
({"max_tokens": 32, "max_output_tokens": 64}, 64),
|
||||
({"thinking": {"budget_tokens": 64}}, 256),
|
||||
({"max_tokens": 64, "thinking": {"budget_tokens": 64}}, None),
|
||||
({"thinking": {"budget_tokens": True}}, None),
|
||||
*(({"extra_body": body}, None) for body in (False, list[object](), "invalid")),
|
||||
*(
|
||||
({"extra_body": {key: "override"}}, None)
|
||||
for key in ("input", "messages", "tools", "system", "instructions", "max_tokens", "thinking")
|
||||
),
|
||||
({"extra_body": {"provider_feature_enabled": True}}, 256),
|
||||
),
|
||||
)
|
||||
async def test_output_reservation_and_extra_body_admission(patch: Mapping[str, object], reserve: int | None) -> None:
|
||||
rig: Final = _Rig(local_count=1_750, counts=(1_750, 100))
|
||||
result: Final = await _run(rig, payload={**_payload("responses"), **patch})
|
||||
if reserve is None:
|
||||
assert isinstance(result, core.CompactionFailure)
|
||||
assert rig.local.empty() and rig.native.empty() and rig.calls.empty()
|
||||
elif reserve == 64:
|
||||
assert result is None and rig.calls.empty()
|
||||
else:
|
||||
assert isinstance(result, core.CompactedRequest) and rig.calls.qsize() == 1
|
||||
|
|
@ -13454,6 +13454,14 @@ def _tier_config(**overrides) -> Dict:
|
|||
return {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, **overrides}
|
||||
|
||||
|
||||
class TestContextWindowCompactionConfig:
|
||||
@pytest.mark.parametrize("value", ["", " ", "\t\n", "\u2003", 17, {"model": "summary-model"}])
|
||||
def test_rejects_blank_or_non_string_model_groups(self, value: object) -> None:
|
||||
with pytest.raises(ValidationError, match="context_window_compaction_model"):
|
||||
ComplexityRouterConfig.model_validate(
|
||||
{"tiers": {"SIMPLE": "small-model"}, "context_window_compaction_model": value}
|
||||
)
|
||||
|
||||
class TestContextWindowEscalation:
|
||||
"""A tier decided on complexity alone must still hold the prompt, or the provider 400s.
|
||||
|
||||
|
|
@ -13621,6 +13629,38 @@ class TestContextWindowEscalation:
|
|||
assert result.model == "small-model"
|
||||
assert "context_escalated" not in result.routing_decision
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("compaction_model", [None, "summary-model"])
|
||||
@pytest.mark.parametrize("session_affinity", [False, True])
|
||||
async def test_compaction_keeps_the_selected_group_on_overflow(
|
||||
self, compaction_model: str | None, session_affinity: bool
|
||||
) -> None:
|
||||
router: Final = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=_windowed_router(_SMALL, _BIG),
|
||||
complexity_router_config={
|
||||
"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"},
|
||||
"session_affinity": session_affinity,
|
||||
"context_window_compaction_model": compaction_model,
|
||||
},
|
||||
)
|
||||
request_kwargs: Final = {"metadata": {"session_id": "compaction-session"}}
|
||||
context_fit: Final = await router._request_context_fit(_OVERSIZED_TURNS, request_kwargs)
|
||||
assert context_fit.accepts("small-model") is (compaction_model is not None)
|
||||
original: Final = await router.async_pre_routing_hook(
|
||||
model="test-router", request_kwargs=request_kwargs, messages=[{"role": "user", "content": "ok continue"}]
|
||||
)
|
||||
oversized: Final = await router.async_pre_routing_hook(
|
||||
model="test-router", request_kwargs=request_kwargs, messages=_OVERSIZED_TURNS
|
||||
)
|
||||
|
||||
assert original is not None and original.model == "small-model"
|
||||
assert oversized is not None
|
||||
assert oversized.model == (original.model if compaction_model else "big-model")
|
||||
assert oversized.routing_decision.get("context_escalated", False) is (compaction_model is None)
|
||||
if session_affinity:
|
||||
assert oversized.routing_decision["cause"] == "session_affinity_pin"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_out_of_band_system_and_tools_count_against_the_window(self):
|
||||
"""The Claude Code shape that live-testing caught: a tiny ask riding a top-level
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -262,6 +263,39 @@ def test_strategy_router_dependencies(litellm_params, expected):
|
|||
assert tuple((d.model_name, d.role) for d in found) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("classifier_type", ["heuristic", "llm"])
|
||||
@pytest.mark.parametrize("escalation_enabled", [False, True])
|
||||
@pytest.mark.parametrize("compaction_model", [None, "summary-model", "tier-model"])
|
||||
def test_compaction_dependency_is_independent_of_classification_and_escalation(
|
||||
classifier_type: str, escalation_enabled: bool, compaction_model: str | None
|
||||
) -> None:
|
||||
found: Final = strategy_router_dependencies(
|
||||
{
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {"SIMPLE": "tier-model"},
|
||||
"classifier_type": classifier_type,
|
||||
"classifier_llm_config": {"model": "classifier-model"},
|
||||
"enable_context_window_escalation": escalation_enabled,
|
||||
"context_window_compaction_model": compaction_model,
|
||||
},
|
||||
}
|
||||
)
|
||||
assert tuple(dependency.model_name for dependency in found if dependency.role == "compaction") == (
|
||||
(compaction_model,) if compaction_model else ()
|
||||
)
|
||||
assert tuple(dependency.model_name for dependency in found if dependency.role == "tier") == ("tier-model",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", " ", "\t\n", 17, {"model": "summary-model"}])
|
||||
def test_config_write_rejects_invalid_compaction_model(value: object) -> None:
|
||||
violation: Final = validate_complexity_router_config_write(
|
||||
complexity_router_config={"tiers": {"SIMPLE": "tier-model"}, "context_window_compaction_model": value}
|
||||
)
|
||||
assert violation is not None
|
||||
assert "context_window_compaction_model" in violation
|
||||
|
||||
|
||||
def test_complexity_default_model_param_wins_over_the_config_field():
|
||||
"""ComplexityRouter overwrites config.default_model with the litellm_params one, so the
|
||||
config field is dead whenever the param is set and must not be able to red the router."""
|
||||
|
|
@ -333,6 +367,7 @@ def test_complexity_embedding_model_is_a_dependency_only_when_semantic_matching_
|
|||
("tier_boundaries",),
|
||||
("token_thresholds", "dimension_weights"),
|
||||
("reasoning_override_min_score",),
|
||||
("context_window_compaction_model",),
|
||||
("tiers",),
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from litellm.router import (
|
|||
_anthropic_stream_raised_error_status,
|
||||
_anthropic_stream_should_decline_fallback,
|
||||
_anthropic_stream_should_drop_pre_content_ping,
|
||||
_compaction_deployment_id,
|
||||
_is_retriable_anthropic_status,
|
||||
)
|
||||
from litellm.router_strategy import simple_shuffle
|
||||
|
|
@ -49,6 +50,374 @@ from litellm.types.llms.openai import ChatCompletionRequest
|
|||
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy
|
||||
|
||||
|
||||
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from types import MappingProxyType
|
||||
|
||||
import pytest_asyncio
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.router_strategy.complexity_router.context_compaction import ModelBudget, NativeRequest, native_request_scope
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
||||
_NATIVE_MODELS: Final = {"responses": "openai/gpt-5.6-luna", "messages": "anthropic/claude-sonnet-5"}
|
||||
_NATIVE_PREFIX: Final = (
|
||||
{"role": "user", "content": "historical record " * 5000},
|
||||
{"role": "assistant", "content": "Recorded"},
|
||||
)
|
||||
_NATIVE_CANONICAL: Final = (
|
||||
{"role": "user", "content": "Provider-retained canonical message"},
|
||||
{"type": "compaction", "id": "cmp_native", "encrypted_content": "opaque-native-state"},
|
||||
)
|
||||
_NATIVE_BLOCK: Final = {"type": "compaction", "content": "Native state", "signature": "signed-native-state"}
|
||||
_NATIVE_JSON: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(loop_scope="function")
|
||||
async def overflow_io(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[respx.MockRouter]:
|
||||
from litellm import anthropic_beta_headers_manager
|
||||
from litellm.litellm_core_utils import logging_worker
|
||||
|
||||
worker: Final = logging_worker.LoggingWorker(timeout=0.5)
|
||||
monkeypatch.setattr(
|
||||
anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG",
|
||||
anthropic_beta_headers_manager.GetAnthropicBetaHeadersConfig.load_local_beta_headers_config(),
|
||||
)
|
||||
monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", worker)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
|
||||
for provider in ("openai", "anthropic"):
|
||||
monkeypatch.delenv(f"{provider.upper()}_API_KEY", raising=False)
|
||||
monkeypatch.setattr(litellm, f"{provider}_key", None)
|
||||
monkeypatch.setattr(litellm, "api_key", None)
|
||||
try:
|
||||
with respx.mock(assert_all_called=False) as transport:
|
||||
transport.route(path__regex=r".*/(input_tokens|count_tokens)$").mock(side_effect=_native_wire_response)
|
||||
transport.route(
|
||||
host="native-compact.invalid", path__regex=r"/v1/(responses/compact|messages)$"
|
||||
).mock(side_effect=_native_wire_response)
|
||||
yield transport
|
||||
await asyncio.wait_for(worker.flush(), timeout=5)
|
||||
finally:
|
||||
await worker.stop()
|
||||
|
||||
|
||||
def _native_history(protocol: str) -> list[Mapping[str, object]]:
|
||||
tail: Final = (
|
||||
(
|
||||
{"type": "function_call", "call_id": "call_lookup", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_lookup", "output": "MARIGOLD-482"},
|
||||
)
|
||||
if protocol == "responses"
|
||||
else (
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "lookup", "name": "lookup", "input": {}}]},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "lookup", "content": "MARIGOLD-482"}]},
|
||||
)
|
||||
)
|
||||
return [*_NATIVE_PREFIX, {"role": "user", "content": "What is the launch code?"}, *tail]
|
||||
|
||||
|
||||
def _native_router(
|
||||
protocol: str, *, scenario: str = "normal", pre_checks: bool = True, backup_limit: int = 64000
|
||||
) -> Router:
|
||||
wire_model: Final = _NATIVE_MODELS[protocol]
|
||||
deployments: Final = (
|
||||
("target", "a", 4096, wire_model), ("target", "b", 4096, wire_model),
|
||||
("backup", "backup", backup_limit, wire_model),
|
||||
("compact", "unfit" if scenario == "count" else "compact", 32000, wire_model),
|
||||
("compact", "small", 4096, wire_model),
|
||||
("compact", "foreign", 64000, _NATIVE_MODELS["messages" if protocol == "responses" else "responses"]),
|
||||
("compact", "denied", 0, wire_model),
|
||||
)
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "native-auto",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {tier: "native-target" for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")},
|
||||
"context_window_compaction_model": "native-compact",
|
||||
"max_tokens_from_tier_model": False,
|
||||
"deployment_affinity": True,
|
||||
},
|
||||
},
|
||||
},
|
||||
*(
|
||||
{
|
||||
"model_name": f"native-{group}",
|
||||
"litellm_params": {
|
||||
"model": model,
|
||||
"api_base": f"https://native-{host}.invalid" + ("/v1" if model.startswith("openai/") else ""),
|
||||
"api_key": "" if scenario == "credentials" and host == "compact" else f"test-{host}",
|
||||
"region_name": "us" if scenario == "region" and group == "compact" else "eu",
|
||||
**({"max_tokens": 32000} if scenario == "max-default" and host == "compact" else {}),
|
||||
**(
|
||||
{"instructions" if protocol == "responses" else "system": "Injected default"}
|
||||
if scenario == "prompt-default" and host == "compact" else {}
|
||||
),
|
||||
},
|
||||
"model_info": {
|
||||
"id": host, "max_input_tokens": limit, "max_output_tokens": 8192,
|
||||
"access_groups": (
|
||||
["denied" if host == "denied" else "compact-access"] if group == "compact" else []
|
||||
),
|
||||
},
|
||||
}
|
||||
for group, host, limit, model in deployments
|
||||
),
|
||||
],
|
||||
num_retries=0,
|
||||
enable_pre_call_checks=pre_checks,
|
||||
fallbacks=[{"native-target": ["native-backup"]}],
|
||||
)
|
||||
|
||||
|
||||
def _native_wire_response(request: httpx.Request) -> httpx.Response:
|
||||
body: Final = _NATIVE_JSON.validate_json(request.content)
|
||||
host: Final = request.url.host.removeprefix("native-").removesuffix(".invalid")
|
||||
protocol: Final = "messages" if "/messages" in request.url.path else "responses"
|
||||
credential: Final = request.headers.get("x-api-key") or request.headers.get("authorization", "").removeprefix(
|
||||
"Bearer "
|
||||
)
|
||||
assert credential == f"test-{host}"
|
||||
if request.url.path.endswith(("/input_tokens", "/count_tokens")):
|
||||
tokens: Final = 40000 if host == "unfit" else 12000 if "historical record" in json.dumps(body) else 128
|
||||
return httpx.Response(200, json={"input_tokens": tokens})
|
||||
compact: Final = request.url.path.endswith("/compact") or body.get("compaction") == {"type": "summarize"}
|
||||
if host == "compact":
|
||||
assert compact, tuple(body)
|
||||
if protocol == "responses" and compact:
|
||||
assert not set(body).difference(
|
||||
{"model", "input", "instructions", "previous_response_id", "prompt_cache_key", "prompt_cache_retention"}
|
||||
)
|
||||
response: Final = (
|
||||
{
|
||||
"id": "cmp_native" if compact else "resp_native", "created_at": 1, "model": body["model"],
|
||||
"object": "response.compaction" if compact else "response",
|
||||
"output": list(_NATIVE_CANONICAL) if compact else [{
|
||||
"id": "msg_native", "type": "message", "role": "assistant", "status": "completed",
|
||||
"content": [{"type": "output_text", "text": "MARIGOLD-482", "annotations": []}],
|
||||
}],
|
||||
"usage": {
|
||||
"input_tokens": 12000 if compact else 128, "output_tokens": 8, "total_tokens": 12008 if compact else 136
|
||||
},
|
||||
}
|
||||
if protocol == "responses" else {
|
||||
"id": "msg_native", "type": "message", "role": "assistant", "model": body["model"],
|
||||
"content": [_NATIVE_BLOCK] if compact else [{"type": "text", "text": "MARIGOLD-482"}],
|
||||
"stop_reason": "compaction" if compact else "end_turn",
|
||||
"usage": {
|
||||
"input_tokens": 0, "output_tokens": 0,
|
||||
"iterations": [{"type": "compaction", "input_tokens": 12000, "output_tokens": 8}],
|
||||
} if compact else {"input_tokens": 128, "output_tokens": 8},
|
||||
}
|
||||
)
|
||||
events: Final = (
|
||||
(
|
||||
{"type": "response.created", "sequence_number": 0, "response": {**response, "output": []}},
|
||||
{"type": "response.output_text.delta", "sequence_number": 1, "item_id": "msg_native",
|
||||
"output_index": 0, "content_index": 0, "delta": "MARIGOLD-482"},
|
||||
{"type": "response.completed", "sequence_number": 2, "response": response},
|
||||
)
|
||||
if protocol == "responses" else (
|
||||
{"type": "message_start", "message": {**response, "content": [], "stop_reason": None}},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "MARIGOLD-482"}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 8}},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
)
|
||||
if not body.get("stream"):
|
||||
return httpx.Response(200, json=response)
|
||||
return httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"},
|
||||
content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events),
|
||||
)
|
||||
|
||||
|
||||
async def _native_call(
|
||||
router: Router,
|
||||
protocol: str,
|
||||
history: Sequence[Mapping[str, object]],
|
||||
params: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> object:
|
||||
kwargs: Final = {
|
||||
"metadata": {"session_id": "native-pin", "user_api_key_auth": UserAPIKeyAuth(team_models=["compact-access"])},
|
||||
**params,
|
||||
}
|
||||
if protocol == "responses":
|
||||
return await router.aresponses(model="native-auto", input=list(history), max_output_tokens=128, **kwargs)
|
||||
if protocol == "chat":
|
||||
messages: Final = TypeAdapter(list[AllMessageValues]).validate_python(history)
|
||||
return await router.acompletion(model="native-auto", messages=messages, max_tokens=128, **kwargs)
|
||||
return await router.aanthropic_messages(model="native-auto", messages=list(history), max_tokens=128, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autorouter_native_unarmed_helpers() -> None:
|
||||
router: Final = Router(model_list=[])
|
||||
deployment: Final[DeploymentTypedDict] = {
|
||||
"model_name": "target", "litellm_params": {"model": "openai/gpt-5.6-luna"},
|
||||
"model_info": {"id": "selected", "max_input_tokens": 4096, "max_output_tokens": 128},
|
||||
}
|
||||
assert _compaction_deployment_id(deployment) == "selected"
|
||||
assert _compaction_deployment_id({}) == ""
|
||||
assert _compaction_deployment_id({"model_info": {}}) == ""
|
||||
assert _compaction_deployment_id({"model_info": {"id": 123}}) == ""
|
||||
budget: Final = router._compaction_model_budget(deployment)
|
||||
assert isinstance(budget, ModelBudget) and budget.deployment_id == "selected"
|
||||
payload: Final = router._compaction_request_payload(deployment, {"input": "Hello"})
|
||||
assert payload["input"] == "Hello" and await router._acount_compaction_tokens(budget.model, payload) > 0
|
||||
assert await router._aprepare_selected_deployment(
|
||||
deployment, "target", {"_context_window_compaction_state": {"model": "forged"}}
|
||||
) is None
|
||||
selected: Final = Router(model_list=[deployment])
|
||||
native: Final = NativeRequest("responses", {}, frozenset({"unavailable"}))
|
||||
with native_request_scope(native), pytest.raises(litellm.ContextWindowExceededError, match="No eligible"):
|
||||
selected._common_checks_available_deployment("target", request_kwargs={})
|
||||
legacy: Final = Router(model_list=_native_router("responses").model_list, routing_strategy="usage-based-routing")
|
||||
with pytest.raises(litellm.ContextWindowExceededError, match="legacy synchronous deployment selection"):
|
||||
await legacy.async_get_available_deployment(model="native-auto", request_kwargs={})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
@pytest.mark.parametrize("pre_checks", (False, True))
|
||||
async def test_autorouter_native_pinned_wire(
|
||||
protocol: str, stream: bool, pre_checks: bool, overflow_io: respx.MockRouter
|
||||
) -> None:
|
||||
router: Final = _native_router(protocol, pre_checks=pre_checks, scenario="max-default" if pre_checks else "normal")
|
||||
history: Final = _native_history(protocol)
|
||||
original: Final = copy.deepcopy(history)
|
||||
parameters: Final = {"type": "object", "properties": {}}
|
||||
context: Final = {
|
||||
"instructions" if protocol == "responses" else "system": "Retain request instructions",
|
||||
"tools": [
|
||||
{"type": "function", "name": "lookup", "parameters": parameters}
|
||||
if protocol == "responses" else {"name": "lookup", "input_schema": parameters}
|
||||
],
|
||||
}
|
||||
targets: Final = overflow_io.route(
|
||||
host__regex=r"native-[ab]\.invalid", path__regex=r"/v1/(responses|messages)$"
|
||||
).mock(side_effect=_native_wire_response)
|
||||
for request_history in ([{"role": "user", "content": "Hello"}], history):
|
||||
result: Final = await _native_call(
|
||||
router, protocol, request_history, {**context, "stream": stream, "disable_fallbacks": True}
|
||||
)
|
||||
if isinstance(result, AsyncIterator):
|
||||
assert [chunk async for chunk in result]
|
||||
assert sum(call.request.url.host == "native-compact.invalid" for call in overflow_io.calls) == (
|
||||
2 if request_history is history else 0
|
||||
)
|
||||
assert targets.call_count == 2 and targets.calls[0].request.url.host == targets.calls[1].request.url.host
|
||||
field: Final = "input" if protocol == "responses" else "messages"
|
||||
canonical: Final = (
|
||||
list(_NATIVE_CANONICAL) if protocol == "responses" else [{"role": "assistant", "content": [_NATIVE_BLOCK]}]
|
||||
)
|
||||
sent: Final = _NATIVE_JSON.validate_json(targets.calls[-1].request.content)
|
||||
assert sent[field] == [*canonical, *history[2:]] and history == original
|
||||
assert all(sent[key] == value for key, value in context.items())
|
||||
assert "_context_window_compaction_state" not in sent
|
||||
calls: Final = tuple(call.request for call in overflow_io.calls)
|
||||
compact: Final = next(
|
||||
request for request in calls if request.url.host == "native-compact.invalid"
|
||||
and not request.url.path.endswith(("/input_tokens", "/count_tokens"))
|
||||
)
|
||||
compact_payload: Final = _NATIVE_JSON.validate_json(compact.content)
|
||||
assert compact_payload[field] == history[:2]
|
||||
assert compact_payload.get("tools") == (context["tools"] if protocol == "messages" else None)
|
||||
assert {request.url.host for request in calls} == {targets.calls[0].request.url.host, "native-compact.invalid"}
|
||||
counted: Final = [request for request in calls if request.url.path.endswith(("/input_tokens", "/count_tokens"))]
|
||||
assert len(counted) == 3 and _NATIVE_JSON.validate_json(counted[-1].content)[field] == sent[field]
|
||||
assert all(_NATIVE_JSON.validate_json(counted[-1].content)[key] == sent[key] for key in context)
|
||||
if protocol == "messages":
|
||||
assert compact_payload["max_tokens"] == 4096
|
||||
assert compact_payload["compaction"] == {"type": "summarize"}
|
||||
assert compact_payload["stream"] is False and compact_payload["system"] == context["system"]
|
||||
for request in (*counted, compact, targets.calls[-1].request):
|
||||
assert "compact-2026-09-04" in request.headers.get("anthropic-beta", ""), str(request.url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
@pytest.mark.parametrize("backup_limit", (4096, 64000))
|
||||
async def test_autorouter_native_retry_fallback_original_history(
|
||||
protocol: str, stream: bool, backup_limit: int, overflow_io: respx.MockRouter
|
||||
) -> None:
|
||||
error: Final = {
|
||||
"type": "error", "sequence_number": 0,
|
||||
"error": {"type": "server_error" if protocol == "responses" else "overloaded_error" if stream else "api_error",
|
||||
"code": "server_error", "message": "retry"},
|
||||
}
|
||||
targets: Final = overflow_io.route(
|
||||
host__regex=r"native-[ab]\.invalid", path__regex=r"/v1/(responses|messages)$"
|
||||
).respond(
|
||||
200 if stream else 503 if protocol == "responses" else 500,
|
||||
headers={"content-type": "text/event-stream" if stream else "application/json"},
|
||||
content=f"event: error\ndata: {json.dumps(error)}\n\n" if stream else json.dumps(error),
|
||||
)
|
||||
backup: Final = overflow_io.route(host="native-backup.invalid", path__regex=r"/v1/(responses|messages)$").mock(
|
||||
side_effect=_native_wire_response
|
||||
)
|
||||
history: Final = _native_history(protocol)
|
||||
result: Final = await _native_call(
|
||||
_native_router(protocol, backup_limit=backup_limit), protocol, history,
|
||||
{"stream": stream, "num_retries": int(not stream), "max_retries": 0,
|
||||
"model_group_retry_policy": {
|
||||
"native-auto": RetryPolicy(ServiceUnavailableErrorRetries=1, InternalServerErrorRetries=1)
|
||||
}},
|
||||
)
|
||||
if isinstance(result, AsyncIterator):
|
||||
assert [chunk async for chunk in result]
|
||||
field: Final = "input" if protocol == "responses" else "messages"
|
||||
compacted: Final = _NATIVE_JSON.validate_json(targets.calls[0].request.content)[field]
|
||||
assert targets.call_count == (1 if stream else 2) and backup.call_count == 1
|
||||
assert _NATIVE_JSON.validate_json(targets.calls[-1].request.content)[field] == compacted
|
||||
assert _NATIVE_JSON.validate_json(backup.calls[0].request.content)[field] == (
|
||||
history if backup_limit == 64000 else compacted
|
||||
)
|
||||
assert sum(call.request.url.host == "native-compact.invalid" for call in overflow_io.calls) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ("responses", "messages"))
|
||||
@pytest.mark.parametrize("reason", ("credentials", "region", "prompt-default", "count", "access"))
|
||||
async def test_autorouter_native_compactor_admission(
|
||||
protocol: str, reason: str, overflow_io: respx.MockRouter
|
||||
) -> None:
|
||||
router: Final = _native_router(protocol, scenario=reason)
|
||||
params: Final = {
|
||||
"disable_fallbacks": True,
|
||||
**({"allowed_model_region": "eu"} if reason == "region" else {}),
|
||||
**(
|
||||
{"metadata": {"user_api_key_auth": UserAPIKeyAuth(team_models=["unrelated"])}}
|
||||
if reason == "access" else {}
|
||||
),
|
||||
}
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
await _native_call(router, protocol, _native_history(protocol), params)
|
||||
assert all(call.request.url.path.endswith(("/input_tokens", "/count_tokens")) for call in overflow_io.calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"protocol,provider", (("chat", "responses"), ("responses", "messages"), ("messages", "responses"))
|
||||
)
|
||||
async def test_autorouter_native_unsupported_overflow(
|
||||
protocol: str, provider: str, overflow_io: respx.MockRouter
|
||||
) -> None:
|
||||
history: Final = [*_NATIVE_PREFIX, {"role": "user", "content": "Hello"}]
|
||||
with pytest.raises(litellm.ContextWindowExceededError, match="unsupported"):
|
||||
await _native_call(_native_router(provider), protocol, history, {"disable_fallbacks": True})
|
||||
assert not overflow_io.calls
|
||||
|
||||
|
||||
def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
|
||||
# initialize a real Router (env‑vars can be empty)
|
||||
router = litellm.Router(
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -35905,6 +35905,11 @@ export interface components {
|
|||
* @description Keywords indicating code-related content
|
||||
*/
|
||||
code_keywords?: string[] | null;
|
||||
/**
|
||||
* Context Window Compaction Model
|
||||
* @description Ordinary Router model group used for provider-native compaction when an asynchronous Responses/OpenAI or Messages/Anthropic request exceeds its selected deployment's input budget. The compactor must fit the input and its native output must fit the unchanged answering deployment. Unsupported overflow routes fail explicitly. Compaction takes precedence over context window escalation and incurs a separate billable call under normal access and limits. Nested Auto Routers are not supported. Omit or set null to keep existing routing behavior.
|
||||
*/
|
||||
context_window_compaction_model?: string | null;
|
||||
/**
|
||||
* Context Window Escalation Buffer
|
||||
* @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue