diff --git a/litellm/constants.py b/litellm/constants.py index d62cad74a36..6cca7e7a66f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -143,6 +143,7 @@ DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL: Final = str( os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL", "text-embedding-3-small") ) DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = int(os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10)) +DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT: Final = int(os.getenv("DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT", 12)) DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float( os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3) ) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 06e157498aa..000daf07b25 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11834,6 +11834,13 @@ "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.", "title": "Fail On Error" }, + "frozen_message_count": { + "default": 12, + "description": "Number of leading messages never sent for compression when the request carries no cache_control breakpoint, so the prompt prefix stays byte-identical across turns for providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables.", + "minimum": 0.0, + "title": "Frozen Message Count", + "type": "integer" + }, "grounding_check": { "anyOf": [ { diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py index b7e4d9275fa..5e1069b6e62 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py @@ -37,6 +37,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> unreachable_fallback=litellm_params.unreachable_fallback, timeout=litellm_params.timeout, ccr_retrieval=litellm_params.ccr_retrieval, + frozen_message_count=litellm_params.frozen_message_count, ) litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType] _callback diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index fa113aa4d33..de19cdffb6a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -16,8 +16,14 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger -from litellm.compression.compress import get_protected_indices -from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS +from litellm.compression.compress import ( + _message_has_cache_control, # pyright: ignore[reportPrivateUsage] # shared compression policy helper + get_protected_indices, +) +from litellm.constants import ( + DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT, + HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, +) from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -228,8 +234,16 @@ def _retrieval_result_indices( ) +def _frozen_prefix_indices(messages: Sequence[Mapping[str, object]], frozen_message_count: int) -> frozenset[int]: + if any(_message_has_cache_control(message) for message in messages): + return frozenset() + return frozenset(range(min(frozen_message_count, len(messages)))) + + def _protected_indices( - messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() + messages: Sequence[Mapping[str, object]], + extra_retrieve_call_ids: frozenset[str] = frozenset(), + frozen_message_count: int = 0, ) -> frozenset[int]: """Indices headroom must not send to the compression service. @@ -247,8 +261,10 @@ def _protected_indices( so the model's own earlier tables came back rewritten and it imitated the shape. The tool results those turns asked for stay compressible. """ - protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices( - messages, extra_retrieve_call_ids + protected: Final = ( + frozenset(get_protected_indices(messages)) + | _retrieval_result_indices(messages, extra_retrieve_call_ids) + | _frozen_prefix_indices(messages, frozen_message_count) ) return ( protected @@ -495,6 +511,7 @@ class HeadroomGuardrail(CustomGuardrail): unreachable_fallback: str | None = None, timeout: float | None = None, ccr_retrieval: bool = True, + frozen_message_count: int = DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT, ): self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/") if not self.headroom_api_base: @@ -509,6 +526,7 @@ class HeadroomGuardrail(CustomGuardrail): ) self.timeout: httpx.Timeout = self._resolve_timeout(timeout) self.ccr_retrieval = ccr_retrieval + self.frozen_message_count = frozen_message_count self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) @@ -803,7 +821,7 @@ class HeadroomGuardrail(CustomGuardrail): # reading the untranslated messages so long tool names can be recovered. raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) - protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) + protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids, self.frozen_message_count) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] if not compressible: return inputs diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py index 0df0f0c80ef..234af318772 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py @@ -2,6 +2,8 @@ from typing import Literal from pydantic import BaseModel, Field +from litellm.constants import DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT + from .base import GuardrailConfigModel @@ -30,6 +32,15 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]): default=True, description="Inject the Headroom retrieval tool for hashes declared by the compression service.", ) + frozen_message_count: int = Field( + default=DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT, + ge=0, + description=( + "Number of leading messages never sent for compression when the request carries no " + "cache_control breakpoint, so the prompt prefix stays byte-identical across turns for " + "providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables." + ), + ) @staticmethod def ui_friendly_name() -> str: diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index bcecb5b27db..a70684d16af 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -87,6 +87,7 @@ def _make_guardrail(**kwargs) -> HeadroomGuardrail: api_key=FAKE_API_KEY, guardrail_name="headroom", default_on=True, + frozen_message_count=0, ) defaults.update(kwargs) return HeadroomGuardrail(**defaults) @@ -2435,20 +2436,30 @@ AGENTIC_MESSAGES = [ ] -async def _wire_and_result(guardrail: HeadroomGuardrail, messages: list, returned: list | None = None): +async def _wire_and_result( + guardrail: HeadroomGuardrail, + messages: list, + returned: list | None = None, + compress: bool = False, + request_data: dict | None = None, +): inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))) sent: dict = {} def _echo(**kwargs): sent["messages"] = kwargs["json"]["messages"] - return _make_compress_response( - returned if returned is not None else json.loads(json.dumps(kwargs["json"]["messages"])) - ) + if returned is not None: + service_rows = returned + elif compress: + service_rows = [{**row, "content": "COMPRESSED"} for row in sent["messages"]] + else: + service_rows = json.loads(json.dumps(sent["messages"])) + return _make_compress_response(service_rows) with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): result = await guardrail.apply_guardrail( inputs=inputs, - request_data={"model": "claude-sonnet-4-5-20250929"}, + request_data=request_data or {"model": "claude-sonnet-4-5-20250929"}, input_type="request", ) return sent["messages"], result @@ -2571,6 +2582,127 @@ async def test_mid_history_cache_control_row_is_never_sent_for_compression(guard assert result["structured_messages"][3] == cached_row +# --------------------------------------------------------------------------- +# LIT-8202: implicit prefix caching. Requests without a cache_control marker +# (OpenAI-family, xAI, bedrock_mantle/openai.*) have no explicit breakpoint, so +# the leading rows are frozen by count instead: re-compressing them with +# position-dependent output would change the prefix bytes every turn and turn +# every would-be cache read into a cache write. +# --------------------------------------------------------------------------- + +IMPLICIT_CACHE_TURN_ONE = [ + {"role": "system", "content": "You are a helpful assistant. " + "S" * 5000}, + {"role": "user", "content": "U1 " + "Q" * 5000}, + { + "role": "assistant", + "content": "Reading the file now.", + "tool_calls": [{"id": "t1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "t1", "content": "F1 " + "F" * 5000}, + {"role": "user", "content": "U2 " + "Q" * 5000}, + { + "role": "assistant", + "content": "Listing now.", + "tool_calls": [{"id": "t2", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "t2", "content": "F2 " + "F" * 5000}, + {"role": "user", "content": "U3 " + "Q" * 5000}, + { + "role": "assistant", + "content": "Grepping now.", + "tool_calls": [{"id": "t3", "type": "function", "function": {"name": "Grep", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "t3", "content": "F3 " + "F" * 5000}, + {"role": "user", "content": "U4 " + "Q" * 5000}, + {"role": "assistant", "content": "Done reading."}, + {"role": "user", "content": "U5 " + "Q" * 5000}, + {"role": "assistant", "content": "Done."}, + {"role": "user", "content": "live instruction"}, +] +IMPLICIT_CACHE_TURN_TWO = IMPLICIT_CACHE_TURN_ONE + [ + {"role": "assistant", "content": "Done."}, + {"role": "user", "content": "next instruction"}, +] +GPT_REQUEST_DATA = {"model": "gpt-5.6"} + + +@pytest.mark.asyncio +async def test_leading_prefix_is_byte_identical_across_turns_without_cache_control(): + """Rows 0..11 never reach the service on either turn, so the leading bytes a + provider's implicit prefix cache keys on stay identical as history grows.""" + guardrail = HeadroomGuardrail( + api_base=FAKE_API_BASE, api_key=FAKE_API_KEY, guardrail_name="headroom", default_on=True + ) + wire_one, result_one = await _wire_and_result( + guardrail, IMPLICIT_CACHE_TURN_ONE, compress=True, request_data=GPT_REQUEST_DATA + ) + wire_two, result_two = await _wire_and_result( + guardrail, IMPLICIT_CACHE_TURN_TWO, compress=True, request_data=GPT_REQUEST_DATA + ) + + assert result_one["structured_messages"][:12] == IMPLICIT_CACHE_TURN_ONE[:12] + assert result_two["structured_messages"][:12] == IMPLICIT_CACHE_TURN_TWO[:12] + assert result_one["structured_messages"][:12] == result_two["structured_messages"][:12] + + for wire in (wire_one, wire_two): + wire_json = json.dumps(wire) + for row in IMPLICIT_CACHE_TURN_ONE[:12]: + assert json.dumps(row)[1:-1] not in wire_json + assert "U1 " not in wire_json + assert "F1 " not in wire_json + + # Negative control: compression still runs on rows past the frozen prefix. + assert IMPLICIT_CACHE_TURN_ONE[12] in wire_one + assert IMPLICIT_CACHE_TURN_ONE[12] in wire_two + assert IMPLICIT_CACHE_TURN_ONE[14] in wire_two + assert result_two["structured_messages"][12]["content"] == "COMPRESSED" + assert result_two["structured_messages"][14]["content"] == "COMPRESSED" + + +@pytest.mark.asyncio +async def test_frozen_message_count_zero_restores_full_history_compression(): + guardrail = _make_guardrail(frozen_message_count=0) + + wire, _result = await _wire_and_result( + guardrail, IMPLICIT_CACHE_TURN_ONE, compress=True, request_data=GPT_REQUEST_DATA + ) + + assert IMPLICIT_CACHE_TURN_ONE[1] in wire + + +@pytest.mark.asyncio +async def test_frozen_prefix_not_applied_when_cache_control_present(): + """An explicit breakpoint keeps the existing cache-prefix policy: the frozen + count does not protect rows beyond it.""" + guardrail = _make_guardrail(frozen_message_count=12) + messages = json.loads(json.dumps(IMPLICIT_CACHE_TURN_ONE)) + messages[2]["cache_control"] = {"type": "ephemeral"} + + wire, _result = await _wire_and_result(guardrail, messages, compress=True, request_data=GPT_REQUEST_DATA) + + for index in (4, 7, 10): + assert IMPLICIT_CACHE_TURN_ONE[index] in wire + + +@pytest.mark.asyncio +async def test_frozen_message_count_flows_through_initialize_guardrail(monkeypatch: pytest.MonkeyPatch): + from litellm.constants import DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT + from litellm.proxy.guardrails.guardrail_hooks.headroom import initialize_guardrail + from litellm.types.guardrails import LitellmParams + + monkeypatch.setattr(litellm.logging_callback_manager, "add_litellm_callback", lambda callback: None) + + params = LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE, frozen_message_count=3) + guardrail = initialize_guardrail(params, {"guardrail_name": "headroom", "litellm_params": params}) + assert guardrail.frozen_message_count == 3 + + default_params = LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE) + default_guardrail = initialize_guardrail( + default_params, {"guardrail_name": "headroom", "litellm_params": default_params} + ) + assert default_guardrail.frozen_message_count == DEFAULT_HEADROOM_FROZEN_MESSAGE_COUNT + + # --------------------------------------------------------------------------- # #38558: a client that runs its own tool loop (e.g. Claude Code via the MCP # gateway) executes headroom_retrieve and echoes the recovered original content diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7aa34c5752c..2567be12bf0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32092,6 +32092,12 @@ export interface components { * @default true */ fail_on_error: boolean | null; + /** + * Frozen Message Count + * @description Number of leading messages never sent for compression when the request carries no cache_control breakpoint, so the prompt prefix stays byte-identical across turns for providers with implicit prefix caching (OpenAI, Azure, xAI, Bedrock Mantle). 0 disables. + * @default 12 + */ + frozen_message_count: number; /** * Grounding Check * @description Enable grounding verification to ensure output is grounded in provided context.