fix(headroom): freeze the request prefix for implicit-caching models

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-09-19 20:48:15 +00:00
parent a93bfdc749
commit afe4b5ac28
8 changed files with 283 additions and 7 deletions

View file

@ -214,6 +214,10 @@ def _message_has_cache_control(message: Mapping[str, object]) -> bool:
return False
def has_cache_control_breakpoint(messages: Sequence[Mapping[str, object]]) -> bool:
return any(_message_has_cache_control(message) for message in messages)
def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
last_breakpoint: Final = max(
(index for index, msg in enumerate(messages) if _message_has_cache_control(msg)),
@ -222,13 +226,17 @@ def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[in
return tuple(range(last_breakpoint + 1))
def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
def get_protected_indices(
messages: Sequence[Mapping[str, object]],
frozen_prefix_count: int = 0,
) -> tuple[int, ...]:
"""
Return indices of messages that must never be compressed:
- All system messages
- The last user message
- The last assistant message
- Every message up to and including the last one carrying an Anthropic cache_control breakpoint
- The first `frozen_prefix_count` messages, the stable prefix an implicit-caching provider keys its cache on
The last user message is what the model is being asked to act on right now,
so compressing it replaces the live instruction with a marker. Compression
@ -240,7 +248,12 @@ def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int
system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system")
last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:]
assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")
return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages)))
frozen_prefix: Final = tuple(range(min(max(frozen_prefix_count, 0), len(messages))))
return tuple(
dict.fromkeys(
system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages) + frozen_prefix
)
)
def _combine_scores(

View file

@ -11927,6 +11927,19 @@
"description": "Enable hallucination detection to detect factual inaccuracies.",
"title": "Hallucinations Check"
},
"implicit_cache_frozen_messages": {
"anyOf": [
{
"minimum": 0.0,
"type": "integer"
},
{
"type": "null"
}
],
"description": "Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.",
"title": "Implicit Cache Frozen Messages"
},
"include_evidence": {
"anyOf": [
{

View file

@ -37,6 +37,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) ->
unreachable_fallback=litellm_params.unreachable_fallback,
timeout=litellm_params.timeout,
ccr_retrieval=litellm_params.ccr_retrieval,
implicit_cache_frozen_messages=litellm_params.implicit_cache_frozen_messages,
)
litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType]
_callback

View file

@ -5,7 +5,7 @@ import math
import re
import time
import uuid
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard
@ -16,7 +16,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.compression.compress import get_protected_indices
from litellm.compression.compress import get_protected_indices, has_cache_control_breakpoint
from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
@ -50,6 +50,7 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.router import Router
from litellm.types.guardrails import LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
@ -69,6 +70,29 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60
_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object])
def _default_llm_router() -> Router | None:
from litellm.proxy.proxy_server import llm_router
return llm_router
def _caches_on_request_prefix(model: str | None, messages: Sequence[Mapping[str, object]]) -> bool:
"""True when the deployment caches implicitly on the exact request prefix.
A request that already carries cache_control breakpoints pins its cache
explicitly, so implicit-prefix protection is skipped for it. Models the
model-cost map does not know are treated as non-caching rather than
freezing every conversation forever.
"""
if model is None or has_cache_control_breakpoint(messages):
return False
try:
return litellm.utils.supports_prompt_caching(model)
except Exception:
verbose_proxy_logger.debug("Headroom: supports_prompt_caching lookup failed for model %s", model)
return False
def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip
return isinstance(value, dict)
@ -229,7 +253,9 @@ def _retrieval_result_indices(
def _protected_indices(
messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset()
messages: Sequence[Mapping[str, object]],
extra_retrieve_call_ids: frozenset[str] = frozenset(),
frozen_prefix_count: int = 0,
) -> frozenset[int]:
"""Indices headroom must not send to the compression service.
@ -247,7 +273,7 @@ def _protected_indices(
so the model's own earlier tables came back rewritten and it imitated the
shape. The tool results those turns asked for stay compressible.
"""
protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices(
protected: Final = frozenset(get_protected_indices(messages, frozen_prefix_count)) | _retrieval_result_indices(
messages, extra_retrieve_call_ids
)
return (
@ -495,6 +521,8 @@ class HeadroomGuardrail(CustomGuardrail):
unreachable_fallback: str | None = None,
timeout: float | None = None,
ccr_retrieval: bool = True,
implicit_cache_frozen_messages: int | None = None,
llm_router_getter: Callable[[], Router | None] | None = None,
):
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
if not self.headroom_api_base:
@ -509,6 +537,8 @@ class HeadroomGuardrail(CustomGuardrail):
)
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
self.ccr_retrieval = ccr_retrieval
self.implicit_cache_frozen_messages = implicit_cache_frozen_messages
self._llm_router_getter: Final = llm_router_getter or _default_llm_router
self.async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback,
)
@ -520,6 +550,21 @@ class HeadroomGuardrail(CustomGuardrail):
supported_event_hooks=list(self.get_supported_event_hooks()),
)
def _frozen_prefix_count(self, request_data: dict, messages: Sequence[Mapping[str, object]]) -> int:
router: Final = self._llm_router_getter()
alias: Final = request_data.get("model")
model: str | None = alias if isinstance(alias, str) else None
if router is not None and isinstance(alias, str):
deployments: Final = router.deployments_for_request(alias, request_data)
deployment_model: Final = deployments[0]["litellm_params"].get("model") if deployments else None
if isinstance(deployment_model, str):
model = deployment_model
if not _caches_on_request_prefix(model, messages):
return 0
if self.implicit_cache_frozen_messages is None:
return len(messages)
return self.implicit_cache_frozen_messages
def _should_bypass(self, request_data: dict) -> bool:
psr: Final = request_data.get("proxy_server_request")
if not _is_str_object_dict(psr):
@ -803,7 +848,11 @@ class HeadroomGuardrail(CustomGuardrail):
# reading the untranslated messages so long tool names can be recovered.
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
protected_indices: Final = _protected_indices(
messages,
raw_retrieve_call_ids,
self._frozen_prefix_count(request_data, messages),
)
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
if not compressible:
return inputs

View file

@ -26,6 +26,11 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
"forwards the request uncompressed instead of blocking it."
),
)
implicit_cache_frozen_messages: int | None = Field(
default=None,
ge=0,
description="Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.",
)
ccr_retrieval: bool = Field(
default=True,
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",

View file

@ -189,6 +189,34 @@ def test_content_that_is_not_a_list_of_mappings_is_not_treated_as_cache_control(
assert sorted(get_protected_indices(messages)) == [0, 2]
def test_frozen_prefix_rows_are_protected():
messages = [
{"role": "system", "content": "sys"},
{"role": "user", "content": "old question"},
{"role": "assistant", "content": "old answer"},
{"role": "tool", "tool_call_id": "t1", "content": "old tool output"},
{"role": "assistant", "content": "ack"},
{"role": "user", "content": "live instruction"},
]
protected = sorted(get_protected_indices(messages, frozen_prefix_count=3))
assert protected == [0, 1, 2, 4, 5]
# frozen_prefix_count=0 must leave the default policy untouched
assert sorted(get_protected_indices(messages)) == [0, 4, 5]
def test_frozen_prefix_count_clamped_to_message_count():
messages = [
{"role": "user", "content": "old question"},
{"role": "assistant", "content": "old answer"},
{"role": "tool", "tool_call_id": "t1", "content": "old tool output"},
{"role": "user", "content": "live instruction"},
]
assert sorted(get_protected_indices(messages, frozen_prefix_count=100)) == [0, 1, 2, 3]
def test_compress_keeps_part_level_cache_control_row_verbatim():
stale_log = {"role": "user", "content": [{"type": "text", "text": "stale log line " * 2000}]}
pinned = {

View file

@ -87,6 +87,9 @@ def _make_guardrail(**kwargs) -> HeadroomGuardrail:
api_key=FAKE_API_KEY,
guardrail_name="headroom",
default_on=True,
# These tests exercise compression mechanics on caching-capable model
# names; the implicit-prefix freeze is opted into explicitly in its own tests.
implicit_cache_frozen_messages=0,
)
defaults.update(kwargs)
return HeadroomGuardrail(**defaults)
@ -3014,3 +3017,162 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float):
assert guardrail.timeout.read == 60.0
assert guardrail.timeout.connect == 5.0
class _StubRouter:
"""Resolves a model alias to a deployment without touching proxy_server."""
def __init__(self, deployment_model: str):
self._deployment_model = deployment_model
def deployments_for_request(self, alias, request_kwargs):
return [{"litellm_params": {"model": self._deployment_model}}]
def _rewrite_every_row(**kwargs):
"""Fake /v1/compress that rewrites every row it is handed to a marker."""
sent = kwargs["json"]["messages"]
rewritten = [{**m, "content": "COMPRESSED"} for m in sent]
return _make_compress_response(rewritten)
def _agentic_request(n_exchanges: int) -> list:
"""An agentic request that is a strict prefix of _agentic_request(n+1)."""
messages = [{"role": "system", "content": "You are an agent."}]
for i in range(1, n_exchanges + 1):
messages.append({"role": "user", "content": f"step {i}"})
messages.append(
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": f"call_{i}", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}
],
}
)
messages.append({"role": "tool", "tool_call_id": f"call_{i}", "content": "log body " + "x" * 2000})
messages.append({"role": "user", "content": f"step {n_exchanges + 1}"})
return messages
async def _apply_two_turns(guardrail: HeadroomGuardrail, turn1: list, turn2: list, model: str):
posts = []
def _record(**kwargs):
posts.append(kwargs["json"]["messages"])
return _rewrite_every_row(**kwargs)
results = []
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record):
for messages in (turn1, turn2):
results.append(
await guardrail.apply_guardrail(
inputs=GenericGuardrailAPIInputs(
texts=["x"], structured_messages=json.loads(json.dumps(messages))
),
request_data={"model": model, "messages": json.loads(json.dumps(messages))},
input_type="request",
)
)
return results, posts
@pytest.mark.asyncio
async def test_implicit_cache_model_freezes_whole_conversation_by_default():
"""LIT-8202: an implicit-caching model keys its cache on the exact request
prefix, so with no explicit freeze count the whole history must be held
back and turn 2 stays a byte-identical append of turn 1's outbound body."""
guardrail = _make_guardrail(
implicit_cache_frozen_messages=None,
llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"),
)
turn1 = _agentic_request(2)
turn2 = _agentic_request(3)
results, posts = await _apply_two_turns(guardrail, turn1, turn2, model="gpt")
assert posts == []
out1 = results[0]["structured_messages"]
out2 = results[1]["structured_messages"]
assert out1 == turn1
assert json.dumps(out2[: len(out1)]) == json.dumps(out1)
@pytest.mark.asyncio
async def test_implicit_cache_frozen_messages_count_freezes_exact_prefix():
"""An explicit count freezes only that many leading rows; later rows are
still handed to the compression service and come back rewritten."""
guardrail = _make_guardrail(
implicit_cache_frozen_messages=2,
llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"),
)
turn1 = _agentic_request(2)
turn2 = _agentic_request(3)
results, posts = await _apply_two_turns(guardrail, turn1, turn2, model="gpt")
assert posts
out1 = results[0]["structured_messages"]
out2 = results[1]["structured_messages"]
assert out1[:2] == turn1[:2]
assert out2[:2] == turn2[:2]
assert json.dumps(out2[:2]) == json.dumps(out1[:2])
assert out1[3]["content"] == "COMPRESSED"
assert [m["role"] for m in posts[0]] == ["tool", "user"]
@pytest.mark.asyncio
async def test_implicit_cache_protection_skipped_when_request_carries_cache_control():
"""A request that pins its own breakpoints keeps the existing cache_control
protection path; the frozen-prefix default must not take over."""
guardrail = _make_guardrail(
implicit_cache_frozen_messages=None,
llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"),
)
turn1 = _agentic_request(2)
turn1[1]["cache_control"] = {"type": "ephemeral"}
request_data = {"model": "gpt", "messages": json.loads(json.dumps(turn1))}
posts = []
def _record(**kwargs):
posts.append(kwargs["json"]["messages"])
return _rewrite_every_row(**kwargs)
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record):
result = await guardrail.apply_guardrail(
inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(turn1))),
request_data=request_data,
input_type="request",
)
assert posts
assert result["structured_messages"][0] == turn1[0]
assert result["structured_messages"][1] == turn1[1]
assert result["structured_messages"][-1] == turn1[-1]
assert result["structured_messages"][3]["content"] == "COMPRESSED"
@pytest.mark.asyncio
async def test_model_without_prompt_caching_keeps_compressing_history():
guardrail = _make_guardrail(
implicit_cache_frozen_messages=None,
llm_router_getter=lambda: _StubRouter("openai/o1-mini"),
)
messages = _agentic_request(2)
posts = []
def _record(**kwargs):
posts.append(kwargs["json"]["messages"])
return _rewrite_every_row(**kwargs)
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record):
result = await guardrail.apply_guardrail(
inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))),
request_data={"model": "o1", "messages": json.loads(json.dumps(messages))},
input_type="request",
)
assert posts
assert result["structured_messages"][3]["content"] == "COMPRESSED"

View file

@ -32132,6 +32132,11 @@ export interface components {
* @description Enable hallucination detection to detect factual inaccuracies.
*/
hallucinations_check?: boolean | null;
/**
* Implicit Cache Frozen Messages
* @description Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.
*/
implicit_cache_frozen_messages?: number | null;
/**
* Include Evidence
* @description Include detailed evidence payloads in responses (sets `plr_evidence` header).