mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(headroom): freeze the leading prompt prefix for implicit-caching providers
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a93bfdc749
commit
90aa1b8349
7 changed files with 187 additions and 11 deletions
|
|
@ -143,6 +143,7 @@ DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL: Final = str(
|
|||
os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL", "text-embedding-3-small")
|
||||
)
|
||||
DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = int(os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10))
|
||||
DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT: Final = int(os.getenv("DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT", 12))
|
||||
DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float(
|
||||
os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -11834,6 +11834,13 @@
|
|||
"description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
|
||||
"title": "Fail On Error"
|
||||
},
|
||||
"frozen_message_count": {
|
||||
"default": 12,
|
||||
"description": "Number of leading messages never sent for compression when the request carries no cache_control breakpoint, so the prompt prefix stays byte-identical across turns for providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables.",
|
||||
"minimum": 0.0,
|
||||
"title": "Frozen Message Count",
|
||||
"type": "integer"
|
||||
},
|
||||
"grounding_check": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) ->
|
|||
unreachable_fallback=litellm_params.unreachable_fallback,
|
||||
timeout=litellm_params.timeout,
|
||||
ccr_retrieval=litellm_params.ccr_retrieval,
|
||||
frozen_message_count=litellm_params.frozen_message_count,
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType]
|
||||
_callback
|
||||
|
|
|
|||
|
|
@ -16,8 +16,14 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.compression.compress import get_protected_indices
|
||||
from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
|
||||
from litellm.compression.compress import (
|
||||
_message_has_cache_control, # pyright: ignore[reportPrivateUsage] # shared compression policy helper
|
||||
get_protected_indices,
|
||||
)
|
||||
from litellm.constants import (
|
||||
DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT,
|
||||
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
|
|
@ -228,8 +234,16 @@ def _retrieval_result_indices(
|
|||
)
|
||||
|
||||
|
||||
def _frozen_prefix_indices(messages: Sequence[Mapping[str, object]], frozen_message_count: int) -> frozenset[int]:
|
||||
if any(_message_has_cache_control(message) for message in messages):
|
||||
return frozenset()
|
||||
return frozenset(range(min(frozen_message_count, len(messages))))
|
||||
|
||||
|
||||
def _protected_indices(
|
||||
messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset()
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
extra_retrieve_call_ids: frozenset[str] = frozenset(),
|
||||
frozen_message_count: int = 0,
|
||||
) -> frozenset[int]:
|
||||
"""Indices headroom must not send to the compression service.
|
||||
|
||||
|
|
@ -247,8 +261,10 @@ def _protected_indices(
|
|||
so the model's own earlier tables came back rewritten and it imitated the
|
||||
shape. The tool results those turns asked for stay compressible.
|
||||
"""
|
||||
protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices(
|
||||
messages, extra_retrieve_call_ids
|
||||
protected: Final = (
|
||||
frozenset(get_protected_indices(messages))
|
||||
| _retrieval_result_indices(messages, extra_retrieve_call_ids)
|
||||
| _frozen_prefix_indices(messages, frozen_message_count)
|
||||
)
|
||||
return (
|
||||
protected
|
||||
|
|
@ -495,6 +511,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
unreachable_fallback: str | None = None,
|
||||
timeout: float | None = None,
|
||||
ccr_retrieval: bool = True,
|
||||
frozen_message_count: int = DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT,
|
||||
):
|
||||
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
|
||||
if not self.headroom_api_base:
|
||||
|
|
@ -509,6 +526,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
)
|
||||
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
|
||||
self.ccr_retrieval = ccr_retrieval
|
||||
self.frozen_message_count = frozen_message_count
|
||||
self.async_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback,
|
||||
)
|
||||
|
|
@ -803,7 +821,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
# reading the untranslated messages so long tool names can be recovered.
|
||||
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
|
||||
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
|
||||
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
|
||||
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids, self.frozen_message_count)
|
||||
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
|
||||
if not compressible:
|
||||
return inputs
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ from typing import Literal
|
|||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from litellm.constants import DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT
|
||||
|
||||
from .base import GuardrailConfigModel
|
||||
|
||||
|
||||
|
|
@ -30,6 +32,15 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
|
|||
default=True,
|
||||
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",
|
||||
)
|
||||
frozen_message_count: int = Field(
|
||||
default=DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT,
|
||||
ge=0,
|
||||
description=(
|
||||
"Number of leading messages never sent for compression when the request carries no "
|
||||
"cache_control breakpoint, so the prompt prefix stays byte-identical across turns for "
|
||||
"providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables."
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ def _make_guardrail(**kwargs) -> HeadroomGuardrail:
|
|||
api_key=FAKE_API_KEY,
|
||||
guardrail_name="headroom",
|
||||
default_on=True,
|
||||
frozen_message_count=0,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return HeadroomGuardrail(**defaults)
|
||||
|
|
@ -2435,20 +2436,30 @@ AGENTIC_MESSAGES = [
|
|||
]
|
||||
|
||||
|
||||
async def _wire_and_result(guardrail: HeadroomGuardrail, messages: list, returned: list | None = None):
|
||||
async def _wire_and_result(
|
||||
guardrail: HeadroomGuardrail,
|
||||
messages: list,
|
||||
returned: list | None = None,
|
||||
compress: bool = False,
|
||||
request_data: dict | None = None,
|
||||
):
|
||||
inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages)))
|
||||
sent: dict = {}
|
||||
|
||||
def _echo(**kwargs):
|
||||
sent["messages"] = kwargs["json"]["messages"]
|
||||
return _make_compress_response(
|
||||
returned if returned is not None else json.loads(json.dumps(kwargs["json"]["messages"]))
|
||||
)
|
||||
if returned is not None:
|
||||
service_rows = returned
|
||||
elif compress:
|
||||
service_rows = [{**row, "content": "COMPRESSED"} for row in sent["messages"]]
|
||||
else:
|
||||
service_rows = json.loads(json.dumps(sent["messages"]))
|
||||
return _make_compress_response(service_rows)
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo):
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={"model": "claude-sonnet-4-5-20250929"},
|
||||
request_data=request_data or {"model": "claude-sonnet-4-5-20250929"},
|
||||
input_type="request",
|
||||
)
|
||||
return sent["messages"], result
|
||||
|
|
@ -2571,6 +2582,127 @@ async def test_mid_history_cache_control_row_is_never_sent_for_compression(guard
|
|||
assert result["structured_messages"][3] == cached_row
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LIT-8202: implicit prefix caching. Requests without a cache_control marker
|
||||
# (OpenAI-family, xAI, bedrock_mantle/openai.*) have no explicit breakpoint, so
|
||||
# the leading rows are frozen by count instead: re-compressing them with
|
||||
# position-dependent output would change the prefix bytes every turn and turn
|
||||
# every would-be cache read into a cache write.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
IMPLICIT_CACHE_TURN_ONE = [
|
||||
{"role": "system", "content": "You are a helpful assistant. " + "S" * 5000},
|
||||
{"role": "user", "content": "U1 " + "Q" * 5000},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Reading the file now.",
|
||||
"tool_calls": [{"id": "t1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": "F1 " + "F" * 5000},
|
||||
{"role": "user", "content": "U2 " + "Q" * 5000},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Listing now.",
|
||||
"tool_calls": [{"id": "t2", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "t2", "content": "F2 " + "F" * 5000},
|
||||
{"role": "user", "content": "U3 " + "Q" * 5000},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Grepping now.",
|
||||
"tool_calls": [{"id": "t3", "type": "function", "function": {"name": "Grep", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "t3", "content": "F3 " + "F" * 5000},
|
||||
{"role": "user", "content": "U4 " + "Q" * 5000},
|
||||
{"role": "assistant", "content": "Done reading."},
|
||||
{"role": "user", "content": "U5 " + "Q" * 5000},
|
||||
{"role": "assistant", "content": "Done."},
|
||||
{"role": "user", "content": "live instruction"},
|
||||
]
|
||||
IMPLICIT_CACHE_TURN_TWO = IMPLICIT_CACHE_TURN_ONE + [
|
||||
{"role": "assistant", "content": "Done."},
|
||||
{"role": "user", "content": "next instruction"},
|
||||
]
|
||||
GPT_REQUEST_DATA = {"model": "gpt-5.6"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_leading_prefix_is_byte_identical_across_turns_without_cache_control():
|
||||
"""Rows 0..11 never reach the service on either turn, so the leading bytes a
|
||||
provider's implicit prefix cache keys on stay identical as history grows."""
|
||||
guardrail = HeadroomGuardrail(
|
||||
api_base=FAKE_API_BASE, api_key=FAKE_API_KEY, guardrail_name="headroom", default_on=True
|
||||
)
|
||||
wire_one, result_one = await _wire_and_result(
|
||||
guardrail, IMPLICIT_CACHE_TURN_ONE, compress=True, request_data=GPT_REQUEST_DATA
|
||||
)
|
||||
wire_two, result_two = await _wire_and_result(
|
||||
guardrail, IMPLICIT_CACHE_TURN_TWO, compress=True, request_data=GPT_REQUEST_DATA
|
||||
)
|
||||
|
||||
assert result_one["structured_messages"][:12] == IMPLICIT_CACHE_TURN_ONE[:12]
|
||||
assert result_two["structured_messages"][:12] == IMPLICIT_CACHE_TURN_TWO[:12]
|
||||
assert result_one["structured_messages"][:12] == result_two["structured_messages"][:12]
|
||||
|
||||
for wire in (wire_one, wire_two):
|
||||
wire_json = json.dumps(wire)
|
||||
for row in IMPLICIT_CACHE_TURN_ONE[:12]:
|
||||
assert json.dumps(row)[1:-1] not in wire_json
|
||||
assert "U1 " not in wire_json
|
||||
assert "F1 " not in wire_json
|
||||
|
||||
# Negative control: compression still runs on rows past the frozen prefix.
|
||||
assert IMPLICIT_CACHE_TURN_ONE[12] in wire_one
|
||||
assert IMPLICIT_CACHE_TURN_ONE[12] in wire_two
|
||||
assert IMPLICIT_CACHE_TURN_ONE[14] in wire_two
|
||||
assert result_two["structured_messages"][12]["content"] == "COMPRESSED"
|
||||
assert result_two["structured_messages"][14]["content"] == "COMPRESSED"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frozen_message_count_zero_restores_full_history_compression():
|
||||
guardrail = _make_guardrail(frozen_message_count=0)
|
||||
|
||||
wire, _result = await _wire_and_result(
|
||||
guardrail, IMPLICIT_CACHE_TURN_ONE, compress=True, request_data=GPT_REQUEST_DATA
|
||||
)
|
||||
|
||||
assert IMPLICIT_CACHE_TURN_ONE[1] in wire
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frozen_prefix_not_applied_when_cache_control_present():
|
||||
"""An explicit breakpoint keeps the existing cache-prefix policy: the frozen
|
||||
count does not protect rows beyond it."""
|
||||
guardrail = _make_guardrail(frozen_message_count=12)
|
||||
messages = json.loads(json.dumps(IMPLICIT_CACHE_TURN_ONE))
|
||||
messages[2]["cache_control"] = {"type": "ephemeral"}
|
||||
|
||||
wire, _result = await _wire_and_result(guardrail, messages, compress=True, request_data=GPT_REQUEST_DATA)
|
||||
|
||||
for index in (4, 7, 10):
|
||||
assert IMPLICIT_CACHE_TURN_ONE[index] in wire
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frozen_message_count_flows_through_initialize_guardrail(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.constants import DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT
|
||||
from litellm.proxy.guardrails.guardrail_hooks.headroom import initialize_guardrail
|
||||
from litellm.types.guardrails import LitellmParams
|
||||
|
||||
monkeypatch.setattr(litellm.logging_callback_manager, "add_litellm_callback", lambda callback: None)
|
||||
|
||||
params = LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE, frozen_message_count=3)
|
||||
guardrail = initialize_guardrail(params, {"guardrail_name": "headroom", "litellm_params": params})
|
||||
assert guardrail.frozen_message_count == 3
|
||||
|
||||
default_params = LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE)
|
||||
default_guardrail = initialize_guardrail(
|
||||
default_params, {"guardrail_name": "headroom", "litellm_params": default_params}
|
||||
)
|
||||
assert default_guardrail.frozen_message_count == DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #38558: a client that runs its own tool loop (e.g. Claude Code via the MCP
|
||||
# gateway) executes headroom_retrieve and echoes the recovered original content
|
||||
|
|
|
|||
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -32092,6 +32092,12 @@ export interface components {
|
|||
* @default true
|
||||
*/
|
||||
fail_on_error: boolean | null;
|
||||
/**
|
||||
* Frozen Message Count
|
||||
* @description Number of leading messages never sent for compression when the request carries no cache_control breakpoint, so the prompt prefix stays byte-identical across turns for providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables.
|
||||
* @default 12
|
||||
*/
|
||||
frozen_message_count: number;
|
||||
/**
|
||||
* Grounding Check
|
||||
* @description Enable grounding verification to ensure output is grounded in provided context.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue