fix(headroom): freeze the leading prompt prefix for implicit-caching providers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-19 20:26:45 +00:00
parent a93bfdc749
commit 90aa1b8349
7 changed files with 187 additions and 11 deletions

View file

@ -143,6 +143,7 @@ DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL: Final = str(
os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL", "text-embedding-3-small")
)
DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = int(os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10))
DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT: Final = int(os.getenv("DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT", 12))
DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float(
os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3)
)

View file

@ -11834,6 +11834,13 @@
"description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
"title": "Fail On Error"
},
"frozen_message_count": {
"default": 12,
"description": "Number of leading messages never sent for compression when the request carries no cache_control breakpoint, so the prompt prefix stays byte-identical across turns for providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables.",
"minimum": 0.0,
"title": "Frozen Message Count",
"type": "integer"
},
"grounding_check": {
"anyOf": [
{

View file

@ -37,6 +37,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) ->
unreachable_fallback=litellm_params.unreachable_fallback,
timeout=litellm_params.timeout,
ccr_retrieval=litellm_params.ccr_retrieval,
frozen_message_count=litellm_params.frozen_message_count,
)
litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType]
_callback

View file

@ -16,8 +16,14 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.compression.compress import get_protected_indices
from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
from litellm.compression.compress import (
_message_has_cache_control, # pyright: ignore[reportPrivateUsage] # shared compression policy helper
get_protected_indices,
)
from litellm.constants import (
DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT,
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
)
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
@ -228,8 +234,16 @@ def _retrieval_result_indices(
)
def _frozen_prefix_indices(messages: Sequence[Mapping[str, object]], frozen_message_count: int) -> frozenset[int]:
if any(_message_has_cache_control(message) for message in messages):
return frozenset()
return frozenset(range(min(frozen_message_count, len(messages))))
def _protected_indices(
messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset()
messages: Sequence[Mapping[str, object]],
extra_retrieve_call_ids: frozenset[str] = frozenset(),
frozen_message_count: int = 0,
) -> frozenset[int]:
"""Indices headroom must not send to the compression service.
@ -247,8 +261,10 @@ def _protected_indices(
so the model's own earlier tables came back rewritten and it imitated the
shape. The tool results those turns asked for stay compressible.
"""
protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices(
messages, extra_retrieve_call_ids
protected: Final = (
frozenset(get_protected_indices(messages))
| _retrieval_result_indices(messages, extra_retrieve_call_ids)
| _frozen_prefix_indices(messages, frozen_message_count)
)
return (
protected
@ -495,6 +511,7 @@ class HeadroomGuardrail(CustomGuardrail):
unreachable_fallback: str | None = None,
timeout: float | None = None,
ccr_retrieval: bool = True,
frozen_message_count: int = DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT,
):
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
if not self.headroom_api_base:
@ -509,6 +526,7 @@ class HeadroomGuardrail(CustomGuardrail):
)
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
self.ccr_retrieval = ccr_retrieval
self.frozen_message_count = frozen_message_count
self.async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback,
)
@ -803,7 +821,7 @@ class HeadroomGuardrail(CustomGuardrail):
# reading the untranslated messages so long tool names can be recovered.
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids, self.frozen_message_count)
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
if not compressible:
return inputs

View file

@ -2,6 +2,8 @@ from typing import Literal
from pydantic import BaseModel, Field
from litellm.constants import DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT
from .base import GuardrailConfigModel
@ -30,6 +32,15 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
default=True,
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",
)
frozen_message_count: int = Field(
default=DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT,
ge=0,
description=(
"Number of leading messages never sent for compression when the request carries no "
"cache_control breakpoint, so the prompt prefix stays byte-identical across turns for "
"providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables."
),
)
@staticmethod
def ui_friendly_name() -> str:

View file

@ -87,6 +87,7 @@ def _make_guardrail(**kwargs) -> HeadroomGuardrail:
api_key=FAKE_API_KEY,
guardrail_name="headroom",
default_on=True,
frozen_message_count=0,
)
defaults.update(kwargs)
return HeadroomGuardrail(**defaults)
@ -2435,20 +2436,30 @@ AGENTIC_MESSAGES = [
]
async def _wire_and_result(guardrail: HeadroomGuardrail, messages: list, returned: list | None = None):
async def _wire_and_result(
guardrail: HeadroomGuardrail,
messages: list,
returned: list | None = None,
compress: bool = False,
request_data: dict | None = None,
):
inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages)))
sent: dict = {}
def _echo(**kwargs):
sent["messages"] = kwargs["json"]["messages"]
return _make_compress_response(
returned if returned is not None else json.loads(json.dumps(kwargs["json"]["messages"]))
)
if returned is not None:
service_rows = returned
elif compress:
service_rows = [{**row, "content": "COMPRESSED"} for row in sent["messages"]]
else:
service_rows = json.loads(json.dumps(sent["messages"]))
return _make_compress_response(service_rows)
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo):
result = await guardrail.apply_guardrail(
inputs=inputs,
request_data={"model": "claude-sonnet-4-5-20250929"},
request_data=request_data or {"model": "claude-sonnet-4-5-20250929"},
input_type="request",
)
return sent["messages"], result
@ -2571,6 +2582,127 @@ async def test_mid_history_cache_control_row_is_never_sent_for_compression(guard
assert result["structured_messages"][3] == cached_row
# ---------------------------------------------------------------------------
# LIT-8202: implicit prefix caching. Requests without a cache_control marker
# (OpenAI-family, xAI, bedrock_mantle/openai.*) have no explicit breakpoint, so
# the leading rows are frozen by count instead: re-compressing them with
# position-dependent output would change the prefix bytes every turn and turn
# every would-be cache read into a cache write.
# ---------------------------------------------------------------------------
IMPLICIT_CACHE_TURN_ONE = [
{"role": "system", "content": "You are a helpful assistant. " + "S" * 5000},
{"role": "user", "content": "U1 " + "Q" * 5000},
{
"role": "assistant",
"content": "Reading the file now.",
"tool_calls": [{"id": "t1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "t1", "content": "F1 " + "F" * 5000},
{"role": "user", "content": "U2 " + "Q" * 5000},
{
"role": "assistant",
"content": "Listing now.",
"tool_calls": [{"id": "t2", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "t2", "content": "F2 " + "F" * 5000},
{"role": "user", "content": "U3 " + "Q" * 5000},
{
"role": "assistant",
"content": "Grepping now.",
"tool_calls": [{"id": "t3", "type": "function", "function": {"name": "Grep", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "t3", "content": "F3 " + "F" * 5000},
{"role": "user", "content": "U4 " + "Q" * 5000},
{"role": "assistant", "content": "Done reading."},
{"role": "user", "content": "U5 " + "Q" * 5000},
{"role": "assistant", "content": "Done."},
{"role": "user", "content": "live instruction"},
]
IMPLICIT_CACHE_TURN_TWO = IMPLICIT_CACHE_TURN_ONE + [
{"role": "assistant", "content": "Done."},
{"role": "user", "content": "next instruction"},
]
GPT_REQUEST_DATA = {"model": "gpt-5.6"}
@pytest.mark.asyncio
async def test_leading_prefix_is_byte_identical_across_turns_without_cache_control():
"""Rows 0..11 never reach the service on either turn, so the leading bytes a
provider's implicit prefix cache keys on stay identical as history grows."""
guardrail = HeadroomGuardrail(
api_base=FAKE_API_BASE, api_key=FAKE_API_KEY, guardrail_name="headroom", default_on=True
)
wire_one, result_one = await _wire_and_result(
guardrail, IMPLICIT_CACHE_TURN_ONE, compress=True, request_data=GPT_REQUEST_DATA
)
wire_two, result_two = await _wire_and_result(
guardrail, IMPLICIT_CACHE_TURN_TWO, compress=True, request_data=GPT_REQUEST_DATA
)
assert result_one["structured_messages"][:12] == IMPLICIT_CACHE_TURN_ONE[:12]
assert result_two["structured_messages"][:12] == IMPLICIT_CACHE_TURN_TWO[:12]
assert result_one["structured_messages"][:12] == result_two["structured_messages"][:12]
for wire in (wire_one, wire_two):
wire_json = json.dumps(wire)
for row in IMPLICIT_CACHE_TURN_ONE[:12]:
assert json.dumps(row)[1:-1] not in wire_json
assert "U1 " not in wire_json
assert "F1 " not in wire_json
# Negative control: compression still runs on rows past the frozen prefix.
assert IMPLICIT_CACHE_TURN_ONE[12] in wire_one
assert IMPLICIT_CACHE_TURN_ONE[12] in wire_two
assert IMPLICIT_CACHE_TURN_ONE[14] in wire_two
assert result_two["structured_messages"][12]["content"] == "COMPRESSED"
assert result_two["structured_messages"][14]["content"] == "COMPRESSED"
@pytest.mark.asyncio
async def test_frozen_message_count_zero_restores_full_history_compression():
guardrail = _make_guardrail(frozen_message_count=0)
wire, _result = await _wire_and_result(
guardrail, IMPLICIT_CACHE_TURN_ONE, compress=True, request_data=GPT_REQUEST_DATA
)
assert IMPLICIT_CACHE_TURN_ONE[1] in wire
@pytest.mark.asyncio
async def test_frozen_prefix_not_applied_when_cache_control_present():
"""An explicit breakpoint keeps the existing cache-prefix policy: the frozen
count does not protect rows beyond it."""
guardrail = _make_guardrail(frozen_message_count=12)
messages = json.loads(json.dumps(IMPLICIT_CACHE_TURN_ONE))
messages[2]["cache_control"] = {"type": "ephemeral"}
wire, _result = await _wire_and_result(guardrail, messages, compress=True, request_data=GPT_REQUEST_DATA)
for index in (4, 7, 10):
assert IMPLICIT_CACHE_TURN_ONE[index] in wire
@pytest.mark.asyncio
async def test_frozen_message_count_flows_through_initialize_guardrail(monkeypatch: pytest.MonkeyPatch):
from litellm.constants import DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT
from litellm.proxy.guardrails.guardrail_hooks.headroom import initialize_guardrail
from litellm.types.guardrails import LitellmParams
monkeypatch.setattr(litellm.logging_callback_manager, "add_litellm_callback", lambda callback: None)
params = LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE, frozen_message_count=3)
guardrail = initialize_guardrail(params, {"guardrail_name": "headroom", "litellm_params": params})
assert guardrail.frozen_message_count == 3
default_params = LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE)
default_guardrail = initialize_guardrail(
default_params, {"guardrail_name": "headroom", "litellm_params": default_params}
)
assert default_guardrail.frozen_message_count == DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT
# ---------------------------------------------------------------------------
# #38558: a client that runs its own tool loop (e.g. Claude Code via the MCP
# gateway) executes headroom_retrieve and echoes the recovered original content

View file

@ -32092,6 +32092,12 @@ export interface components {
* @default true
*/
fail_on_error: boolean | null;
/**
* Frozen Message Count
* @description Number of leading messages never sent for compression when the request carries no cache_control breakpoint, so the prompt prefix stays byte-identical across turns for providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables.
* @default 12
*/
frozen_message_count: number;
/**
* Grounding Check
* @description Enable grounding verification to ensure output is grounded in provided context.