mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(headroom): freeze the request prefix for implicit-caching models
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a93bfdc749
commit
afe4b5ac28
8 changed files with 283 additions and 7 deletions
|
|
@ -214,6 +214,10 @@ def _message_has_cache_control(message: Mapping[str, object]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def has_cache_control_breakpoint(messages: Sequence[Mapping[str, object]]) -> bool:
|
||||
return any(_message_has_cache_control(message) for message in messages)
|
||||
|
||||
|
||||
def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
|
||||
last_breakpoint: Final = max(
|
||||
(index for index, msg in enumerate(messages) if _message_has_cache_control(msg)),
|
||||
|
|
@ -222,13 +226,17 @@ def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[in
|
|||
return tuple(range(last_breakpoint + 1))
|
||||
|
||||
|
||||
def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
|
||||
def get_protected_indices(
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
frozen_prefix_count: int = 0,
|
||||
) -> tuple[int, ...]:
|
||||
"""
|
||||
Return indices of messages that must never be compressed:
|
||||
- All system messages
|
||||
- The last user message
|
||||
- The last assistant message
|
||||
- Every message up to and including the last one carrying an Anthropic cache_control breakpoint
|
||||
- The first `frozen_prefix_count` messages, the stable prefix an implicit-caching provider keys its cache on
|
||||
|
||||
The last user message is what the model is being asked to act on right now,
|
||||
so compressing it replaces the live instruction with a marker. Compression
|
||||
|
|
@ -240,7 +248,12 @@ def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int
|
|||
system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system")
|
||||
last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:]
|
||||
assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")
|
||||
return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages)))
|
||||
frozen_prefix: Final = tuple(range(min(max(frozen_prefix_count, 0), len(messages))))
|
||||
return tuple(
|
||||
dict.fromkeys(
|
||||
system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages) + frozen_prefix
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _combine_scores(
|
||||
|
|
|
|||
|
|
@ -11927,6 +11927,19 @@
|
|||
"description": "Enable hallucination detection to detect factual inaccuracies.",
|
||||
"title": "Hallucinations Check"
|
||||
},
|
||||
"implicit_cache_frozen_messages": {
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.",
|
||||
"title": "Implicit Cache Frozen Messages"
|
||||
},
|
||||
"include_evidence": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) ->
|
|||
unreachable_fallback=litellm_params.unreachable_fallback,
|
||||
timeout=litellm_params.timeout,
|
||||
ccr_retrieval=litellm_params.ccr_retrieval,
|
||||
implicit_cache_frozen_messages=litellm_params.implicit_cache_frozen_messages,
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType]
|
||||
_callback
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import math
|
|||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard
|
||||
|
||||
|
|
@ -16,7 +16,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.compression.compress import get_protected_indices
|
||||
from litellm.compression.compress import get_protected_indices, has_cache_control_breakpoint
|
||||
from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
|
|
@ -50,6 +50,7 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.router import Router
|
||||
from litellm.types.guardrails import LitellmParams
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
|
|
@ -69,6 +70,29 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60
|
|||
_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _default_llm_router() -> Router | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return llm_router
|
||||
|
||||
|
||||
def _caches_on_request_prefix(model: str | None, messages: Sequence[Mapping[str, object]]) -> bool:
|
||||
"""True when the deployment caches implicitly on the exact request prefix.
|
||||
|
||||
A request that already carries cache_control breakpoints pins its cache
|
||||
explicitly, so implicit-prefix protection is skipped for it. Models the
|
||||
model-cost map does not know are treated as non-caching rather than
|
||||
freezing every conversation forever.
|
||||
"""
|
||||
if model is None or has_cache_control_breakpoint(messages):
|
||||
return False
|
||||
try:
|
||||
return litellm.utils.supports_prompt_caching(model)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug("Headroom: supports_prompt_caching lookup failed for model %s", model)
|
||||
return False
|
||||
|
||||
|
||||
def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
|
@ -229,7 +253,9 @@ def _retrieval_result_indices(
|
|||
|
||||
|
||||
def _protected_indices(
|
||||
messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset()
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
extra_retrieve_call_ids: frozenset[str] = frozenset(),
|
||||
frozen_prefix_count: int = 0,
|
||||
) -> frozenset[int]:
|
||||
"""Indices headroom must not send to the compression service.
|
||||
|
||||
|
|
@ -247,7 +273,7 @@ def _protected_indices(
|
|||
so the model's own earlier tables came back rewritten and it imitated the
|
||||
shape. The tool results those turns asked for stay compressible.
|
||||
"""
|
||||
protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices(
|
||||
protected: Final = frozenset(get_protected_indices(messages, frozen_prefix_count)) | _retrieval_result_indices(
|
||||
messages, extra_retrieve_call_ids
|
||||
)
|
||||
return (
|
||||
|
|
@ -495,6 +521,8 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
unreachable_fallback: str | None = None,
|
||||
timeout: float | None = None,
|
||||
ccr_retrieval: bool = True,
|
||||
implicit_cache_frozen_messages: int | None = None,
|
||||
llm_router_getter: Callable[[], Router | None] | None = None,
|
||||
):
|
||||
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
|
||||
if not self.headroom_api_base:
|
||||
|
|
@ -509,6 +537,8 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
)
|
||||
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
|
||||
self.ccr_retrieval = ccr_retrieval
|
||||
self.implicit_cache_frozen_messages = implicit_cache_frozen_messages
|
||||
self._llm_router_getter: Final = llm_router_getter or _default_llm_router
|
||||
self.async_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback,
|
||||
)
|
||||
|
|
@ -520,6 +550,21 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
supported_event_hooks=list(self.get_supported_event_hooks()),
|
||||
)
|
||||
|
||||
def _frozen_prefix_count(self, request_data: dict, messages: Sequence[Mapping[str, object]]) -> int:
|
||||
router: Final = self._llm_router_getter()
|
||||
alias: Final = request_data.get("model")
|
||||
model: str | None = alias if isinstance(alias, str) else None
|
||||
if router is not None and isinstance(alias, str):
|
||||
deployments: Final = router.deployments_for_request(alias, request_data)
|
||||
deployment_model: Final = deployments[0]["litellm_params"].get("model") if deployments else None
|
||||
if isinstance(deployment_model, str):
|
||||
model = deployment_model
|
||||
if not _caches_on_request_prefix(model, messages):
|
||||
return 0
|
||||
if self.implicit_cache_frozen_messages is None:
|
||||
return len(messages)
|
||||
return self.implicit_cache_frozen_messages
|
||||
|
||||
def _should_bypass(self, request_data: dict) -> bool:
|
||||
psr: Final = request_data.get("proxy_server_request")
|
||||
if not _is_str_object_dict(psr):
|
||||
|
|
@ -803,7 +848,11 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
# reading the untranslated messages so long tool names can be recovered.
|
||||
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
|
||||
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
|
||||
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
|
||||
protected_indices: Final = _protected_indices(
|
||||
messages,
|
||||
raw_retrieve_call_ids,
|
||||
self._frozen_prefix_count(request_data, messages),
|
||||
)
|
||||
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
|
||||
if not compressible:
|
||||
return inputs
|
||||
|
|
|
|||
|
|
@ -26,6 +26,11 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
|
|||
"forwards the request uncompressed instead of blocking it."
|
||||
),
|
||||
)
|
||||
implicit_cache_frozen_messages: int | None = Field(
|
||||
default=None,
|
||||
ge=0,
|
||||
description="Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.",
|
||||
)
|
||||
ccr_retrieval: bool = Field(
|
||||
default=True,
|
||||
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",
|
||||
|
|
|
|||
|
|
@ -189,6 +189,34 @@ def test_content_that_is_not_a_list_of_mappings_is_not_treated_as_cache_control(
|
|||
assert sorted(get_protected_indices(messages)) == [0, 2]
|
||||
|
||||
|
||||
def test_frozen_prefix_rows_are_protected():
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old question"},
|
||||
{"role": "assistant", "content": "old answer"},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": "old tool output"},
|
||||
{"role": "assistant", "content": "ack"},
|
||||
{"role": "user", "content": "live instruction"},
|
||||
]
|
||||
|
||||
protected = sorted(get_protected_indices(messages, frozen_prefix_count=3))
|
||||
|
||||
assert protected == [0, 1, 2, 4, 5]
|
||||
# frozen_prefix_count=0 must leave the default policy untouched
|
||||
assert sorted(get_protected_indices(messages)) == [0, 4, 5]
|
||||
|
||||
|
||||
def test_frozen_prefix_count_clamped_to_message_count():
|
||||
messages = [
|
||||
{"role": "user", "content": "old question"},
|
||||
{"role": "assistant", "content": "old answer"},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": "old tool output"},
|
||||
{"role": "user", "content": "live instruction"},
|
||||
]
|
||||
|
||||
assert sorted(get_protected_indices(messages, frozen_prefix_count=100)) == [0, 1, 2, 3]
|
||||
|
||||
|
||||
def test_compress_keeps_part_level_cache_control_row_verbatim():
|
||||
stale_log = {"role": "user", "content": [{"type": "text", "text": "stale log line " * 2000}]}
|
||||
pinned = {
|
||||
|
|
|
|||
|
|
@ -87,6 +87,9 @@ def _make_guardrail(**kwargs) -> HeadroomGuardrail:
|
|||
api_key=FAKE_API_KEY,
|
||||
guardrail_name="headroom",
|
||||
default_on=True,
|
||||
# These tests exercise compression mechanics on caching-capable model
|
||||
# names; the implicit-prefix freeze is opted into explicitly in its own tests.
|
||||
implicit_cache_frozen_messages=0,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return HeadroomGuardrail(**defaults)
|
||||
|
|
@ -3014,3 +3017,162 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float):
|
|||
|
||||
assert guardrail.timeout.read == 60.0
|
||||
assert guardrail.timeout.connect == 5.0
|
||||
|
||||
|
||||
class _StubRouter:
|
||||
"""Resolves a model alias to a deployment without touching proxy_server."""
|
||||
|
||||
def __init__(self, deployment_model: str):
|
||||
self._deployment_model = deployment_model
|
||||
|
||||
def deployments_for_request(self, alias, request_kwargs):
|
||||
return [{"litellm_params": {"model": self._deployment_model}}]
|
||||
|
||||
|
||||
def _rewrite_every_row(**kwargs):
|
||||
"""Fake /v1/compress that rewrites every row it is handed to a marker."""
|
||||
sent = kwargs["json"]["messages"]
|
||||
rewritten = [{**m, "content": "COMPRESSED"} for m in sent]
|
||||
return _make_compress_response(rewritten)
|
||||
|
||||
|
||||
def _agentic_request(n_exchanges: int) -> list:
|
||||
"""An agentic request that is a strict prefix of _agentic_request(n+1)."""
|
||||
messages = [{"role": "system", "content": "You are an agent."}]
|
||||
for i in range(1, n_exchanges + 1):
|
||||
messages.append({"role": "user", "content": f"step {i}"})
|
||||
messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": f"call_{i}", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}
|
||||
],
|
||||
}
|
||||
)
|
||||
messages.append({"role": "tool", "tool_call_id": f"call_{i}", "content": "log body " + "x" * 2000})
|
||||
messages.append({"role": "user", "content": f"step {n_exchanges + 1}"})
|
||||
return messages
|
||||
|
||||
|
||||
async def _apply_two_turns(guardrail: HeadroomGuardrail, turn1: list, turn2: list, model: str):
|
||||
posts = []
|
||||
|
||||
def _record(**kwargs):
|
||||
posts.append(kwargs["json"]["messages"])
|
||||
return _rewrite_every_row(**kwargs)
|
||||
|
||||
results = []
|
||||
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record):
|
||||
for messages in (turn1, turn2):
|
||||
results.append(
|
||||
await guardrail.apply_guardrail(
|
||||
inputs=GenericGuardrailAPIInputs(
|
||||
texts=["x"], structured_messages=json.loads(json.dumps(messages))
|
||||
),
|
||||
request_data={"model": model, "messages": json.loads(json.dumps(messages))},
|
||||
input_type="request",
|
||||
)
|
||||
)
|
||||
return results, posts
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_implicit_cache_model_freezes_whole_conversation_by_default():
|
||||
"""LIT-8202: an implicit-caching model keys its cache on the exact request
|
||||
prefix, so with no explicit freeze count the whole history must be held
|
||||
back and turn 2 stays a byte-identical append of turn 1's outbound body."""
|
||||
guardrail = _make_guardrail(
|
||||
implicit_cache_frozen_messages=None,
|
||||
llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"),
|
||||
)
|
||||
turn1 = _agentic_request(2)
|
||||
turn2 = _agentic_request(3)
|
||||
|
||||
results, posts = await _apply_two_turns(guardrail, turn1, turn2, model="gpt")
|
||||
|
||||
assert posts == []
|
||||
out1 = results[0]["structured_messages"]
|
||||
out2 = results[1]["structured_messages"]
|
||||
assert out1 == turn1
|
||||
assert json.dumps(out2[: len(out1)]) == json.dumps(out1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_implicit_cache_frozen_messages_count_freezes_exact_prefix():
|
||||
"""An explicit count freezes only that many leading rows; later rows are
|
||||
still handed to the compression service and come back rewritten."""
|
||||
guardrail = _make_guardrail(
|
||||
implicit_cache_frozen_messages=2,
|
||||
llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"),
|
||||
)
|
||||
turn1 = _agentic_request(2)
|
||||
turn2 = _agentic_request(3)
|
||||
|
||||
results, posts = await _apply_two_turns(guardrail, turn1, turn2, model="gpt")
|
||||
|
||||
assert posts
|
||||
out1 = results[0]["structured_messages"]
|
||||
out2 = results[1]["structured_messages"]
|
||||
assert out1[:2] == turn1[:2]
|
||||
assert out2[:2] == turn2[:2]
|
||||
assert json.dumps(out2[:2]) == json.dumps(out1[:2])
|
||||
assert out1[3]["content"] == "COMPRESSED"
|
||||
assert [m["role"] for m in posts[0]] == ["tool", "user"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_implicit_cache_protection_skipped_when_request_carries_cache_control():
|
||||
"""A request that pins its own breakpoints keeps the existing cache_control
|
||||
protection path; the frozen-prefix default must not take over."""
|
||||
guardrail = _make_guardrail(
|
||||
implicit_cache_frozen_messages=None,
|
||||
llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"),
|
||||
)
|
||||
turn1 = _agentic_request(2)
|
||||
turn1[1]["cache_control"] = {"type": "ephemeral"}
|
||||
|
||||
request_data = {"model": "gpt", "messages": json.loads(json.dumps(turn1))}
|
||||
posts = []
|
||||
|
||||
def _record(**kwargs):
|
||||
posts.append(kwargs["json"]["messages"])
|
||||
return _rewrite_every_row(**kwargs)
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record):
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(turn1))),
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert posts
|
||||
assert result["structured_messages"][0] == turn1[0]
|
||||
assert result["structured_messages"][1] == turn1[1]
|
||||
assert result["structured_messages"][-1] == turn1[-1]
|
||||
assert result["structured_messages"][3]["content"] == "COMPRESSED"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_without_prompt_caching_keeps_compressing_history():
|
||||
guardrail = _make_guardrail(
|
||||
implicit_cache_frozen_messages=None,
|
||||
llm_router_getter=lambda: _StubRouter("openai/o1-mini"),
|
||||
)
|
||||
messages = _agentic_request(2)
|
||||
|
||||
posts = []
|
||||
|
||||
def _record(**kwargs):
|
||||
posts.append(kwargs["json"]["messages"])
|
||||
return _rewrite_every_row(**kwargs)
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record):
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))),
|
||||
request_data={"model": "o1", "messages": json.loads(json.dumps(messages))},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert posts
|
||||
assert result["structured_messages"][3]["content"] == "COMPRESSED"
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -32132,6 +32132,11 @@ export interface components {
|
|||
* @description Enable hallucination detection to detect factual inaccuracies.
|
||||
*/
|
||||
hallucinations_check?: boolean | null;
|
||||
/**
|
||||
* Implicit Cache Frozen Messages
|
||||
* @description Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.
|
||||
*/
|
||||
implicit_cache_frozen_messages?: number | null;
|
||||
/**
|
||||
* Include Evidence
|
||||
* @description Include detailed evidence payloads in responses (sets `plr_evidence` header).
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue