diff --git a/litellm/compression/compress.py b/litellm/compression/compress.py index bc2cb7c9fdc..bd725d3fe69 100644 --- a/litellm/compression/compress.py +++ b/litellm/compression/compress.py @@ -214,6 +214,10 @@ def _message_has_cache_control(message: Mapping[str, object]) -> bool: return False +def has_cache_control_breakpoint(messages: Sequence[Mapping[str, object]]) -> bool: + return any(_message_has_cache_control(message) for message in messages) + + def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]: last_breakpoint: Final = max( (index for index, msg in enumerate(messages) if _message_has_cache_control(msg)), @@ -222,13 +226,17 @@ def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[in return tuple(range(last_breakpoint + 1)) -def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]: +def get_protected_indices( + messages: Sequence[Mapping[str, object]], + frozen_prefix_count: int = 0, +) -> tuple[int, ...]: """ Return indices of messages that must never be compressed: - All system messages - The last user message - The last assistant message - Every message up to and including the last one carrying an Anthropic cache_control breakpoint + - The first `frozen_prefix_count` messages, the stable prefix an implicit-caching provider keys its cache on The last user message is what the model is being asked to act on right now, so compressing it replaces the live instruction with a marker. Compression @@ -240,7 +248,12 @@ def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system") last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:] assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant") - return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages))) + frozen_prefix: Final = tuple(range(min(max(frozen_prefix_count, 0), len(messages)))) + return tuple( + dict.fromkeys( + system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages) + frozen_prefix + ) + ) def _combine_scores( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 06e157498aa..5b947a6ae07 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11927,6 +11927,19 @@ "description": "Enable hallucination detection to detect factual inaccuracies.", "title": "Hallucinations Check" }, + "implicit_cache_frozen_messages": { + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.", + "title": "Implicit Cache Frozen Messages" + }, "include_evidence": { "anyOf": [ { diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py index b7e4d9275fa..a2e154511ef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py @@ -37,6 +37,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> unreachable_fallback=litellm_params.unreachable_fallback, timeout=litellm_params.timeout, ccr_retrieval=litellm_params.ccr_retrieval, + implicit_cache_frozen_messages=litellm_params.implicit_cache_frozen_messages, ) litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType] _callback diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index fa113aa4d33..4fa1972fccb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -5,7 +5,7 @@ import math import re import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard @@ -16,7 +16,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger -from litellm.compression.compress import get_protected_indices +from litellm.compression.compress import get_protected_indices, has_cache_control_breakpoint from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -50,6 +50,7 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -69,6 +70,29 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 _REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) +def _default_llm_router() -> Router | None: + from litellm.proxy.proxy_server import llm_router + + return llm_router + + +def _caches_on_request_prefix(model: str | None, messages: Sequence[Mapping[str, object]]) -> bool: + """True when the deployment caches implicitly on the exact request prefix. + + A request that already carries cache_control breakpoints pins its cache + explicitly, so implicit-prefix protection is skipped for it. Models the + model-cost map does not know are treated as non-caching rather than + freezing every conversation forever. + """ + if model is None or has_cache_control_breakpoint(messages): + return False + try: + return litellm.utils.supports_prompt_caching(model) + except Exception: + verbose_proxy_logger.debug("Headroom: supports_prompt_caching lookup failed for model %s", model) + return False + + def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip return isinstance(value, dict) @@ -229,7 +253,9 @@ def _retrieval_result_indices( def _protected_indices( - messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() + messages: Sequence[Mapping[str, object]], + extra_retrieve_call_ids: frozenset[str] = frozenset(), + frozen_prefix_count: int = 0, ) -> frozenset[int]: """Indices headroom must not send to the compression service. @@ -247,7 +273,7 @@ def _protected_indices( so the model's own earlier tables came back rewritten and it imitated the shape. The tool results those turns asked for stay compressible. """ - protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices( + protected: Final = frozenset(get_protected_indices(messages, frozen_prefix_count)) | _retrieval_result_indices( messages, extra_retrieve_call_ids ) return ( @@ -495,6 +521,8 @@ class HeadroomGuardrail(CustomGuardrail): unreachable_fallback: str | None = None, timeout: float | None = None, ccr_retrieval: bool = True, + implicit_cache_frozen_messages: int | None = None, + llm_router_getter: Callable[[], Router | None] | None = None, ): self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/") if not self.headroom_api_base: @@ -509,6 +537,8 @@ class HeadroomGuardrail(CustomGuardrail): ) self.timeout: httpx.Timeout = self._resolve_timeout(timeout) self.ccr_retrieval = ccr_retrieval + self.implicit_cache_frozen_messages = implicit_cache_frozen_messages + self._llm_router_getter: Final = llm_router_getter or _default_llm_router self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) @@ -520,6 +550,21 @@ class HeadroomGuardrail(CustomGuardrail): supported_event_hooks=list(self.get_supported_event_hooks()), ) + def _frozen_prefix_count(self, request_data: dict, messages: Sequence[Mapping[str, object]]) -> int: + router: Final = self._llm_router_getter() + alias: Final = request_data.get("model") + model: str | None = alias if isinstance(alias, str) else None + if router is not None and isinstance(alias, str): + deployments: Final = router.deployments_for_request(alias, request_data) + deployment_model: Final = deployments[0]["litellm_params"].get("model") if deployments else None + if isinstance(deployment_model, str): + model = deployment_model + if not _caches_on_request_prefix(model, messages): + return 0 + if self.implicit_cache_frozen_messages is None: + return len(messages) + return self.implicit_cache_frozen_messages + def _should_bypass(self, request_data: dict) -> bool: psr: Final = request_data.get("proxy_server_request") if not _is_str_object_dict(psr): @@ -803,7 +848,11 @@ class HeadroomGuardrail(CustomGuardrail): # reading the untranslated messages so long tool names can be recovered. raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) - protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) + protected_indices: Final = _protected_indices( + messages, + raw_retrieve_call_ids, + self._frozen_prefix_count(request_data, messages), + ) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] if not compressible: return inputs diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py index 0df0f0c80ef..4f810d296f9 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py @@ -26,6 +26,11 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]): "forwards the request uncompressed instead of blocking it." ), ) + implicit_cache_frozen_messages: int | None = Field( + default=None, + ge=0, + description="Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection.", + ) ccr_retrieval: bool = Field( default=True, description="Inject the Headroom retrieval tool for hashes declared by the compression service.", diff --git a/tests/test_litellm/compression/test_compress.py b/tests/test_litellm/compression/test_compress.py index 70281f75196..0a5706e5e23 100644 --- a/tests/test_litellm/compression/test_compress.py +++ b/tests/test_litellm/compression/test_compress.py @@ -189,6 +189,34 @@ def test_content_that_is_not_a_list_of_mappings_is_not_treated_as_cache_control( assert sorted(get_protected_indices(messages)) == [0, 2] +def test_frozen_prefix_rows_are_protected(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + {"role": "tool", "tool_call_id": "t1", "content": "old tool output"}, + {"role": "assistant", "content": "ack"}, + {"role": "user", "content": "live instruction"}, + ] + + protected = sorted(get_protected_indices(messages, frozen_prefix_count=3)) + + assert protected == [0, 1, 2, 4, 5] + # frozen_prefix_count=0 must leave the default policy untouched + assert sorted(get_protected_indices(messages)) == [0, 4, 5] + + +def test_frozen_prefix_count_clamped_to_message_count(): + messages = [ + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + {"role": "tool", "tool_call_id": "t1", "content": "old tool output"}, + {"role": "user", "content": "live instruction"}, + ] + + assert sorted(get_protected_indices(messages, frozen_prefix_count=100)) == [0, 1, 2, 3] + + def test_compress_keeps_part_level_cache_control_row_verbatim(): stale_log = {"role": "user", "content": [{"type": "text", "text": "stale log line " * 2000}]} pinned = { diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index bcecb5b27db..0523edb9883 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -87,6 +87,9 @@ def _make_guardrail(**kwargs) -> HeadroomGuardrail: api_key=FAKE_API_KEY, guardrail_name="headroom", default_on=True, + # These tests exercise compression mechanics on caching-capable model + # names; the implicit-prefix freeze is opted into explicitly in its own tests. + implicit_cache_frozen_messages=0, ) defaults.update(kwargs) return HeadroomGuardrail(**defaults) @@ -3014,3 +3017,162 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float): assert guardrail.timeout.read == 60.0 assert guardrail.timeout.connect == 5.0 + + +class _StubRouter: + """Resolves a model alias to a deployment without touching proxy_server.""" + + def __init__(self, deployment_model: str): + self._deployment_model = deployment_model + + def deployments_for_request(self, alias, request_kwargs): + return [{"litellm_params": {"model": self._deployment_model}}] + + +def _rewrite_every_row(**kwargs): + """Fake /v1/compress that rewrites every row it is handed to a marker.""" + sent = kwargs["json"]["messages"] + rewritten = [{**m, "content": "COMPRESSED"} for m in sent] + return _make_compress_response(rewritten) + + +def _agentic_request(n_exchanges: int) -> list: + """An agentic request that is a strict prefix of _agentic_request(n+1).""" + messages = [{"role": "system", "content": "You are an agent."}] + for i in range(1, n_exchanges + 1): + messages.append({"role": "user", "content": f"step {i}"}) + messages.append( + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": f"call_{i}", "type": "function", "function": {"name": "read_file", "arguments": "{}"}} + ], + } + ) + messages.append({"role": "tool", "tool_call_id": f"call_{i}", "content": "log body " + "x" * 2000}) + messages.append({"role": "user", "content": f"step {n_exchanges + 1}"}) + return messages + + +async def _apply_two_turns(guardrail: HeadroomGuardrail, turn1: list, turn2: list, model: str): + posts = [] + + def _record(**kwargs): + posts.append(kwargs["json"]["messages"]) + return _rewrite_every_row(**kwargs) + + results = [] + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record): + for messages in (turn1, turn2): + results.append( + await guardrail.apply_guardrail( + inputs=GenericGuardrailAPIInputs( + texts=["x"], structured_messages=json.loads(json.dumps(messages)) + ), + request_data={"model": model, "messages": json.loads(json.dumps(messages))}, + input_type="request", + ) + ) + return results, posts + + +@pytest.mark.asyncio +async def test_implicit_cache_model_freezes_whole_conversation_by_default(): + """LIT-8202: an implicit-caching model keys its cache on the exact request + prefix, so with no explicit freeze count the whole history must be held + back and turn 2 stays a byte-identical append of turn 1's outbound body.""" + guardrail = _make_guardrail( + implicit_cache_frozen_messages=None, + llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"), + ) + turn1 = _agentic_request(2) + turn2 = _agentic_request(3) + + results, posts = await _apply_two_turns(guardrail, turn1, turn2, model="gpt") + + assert posts == [] + out1 = results[0]["structured_messages"] + out2 = results[1]["structured_messages"] + assert out1 == turn1 + assert json.dumps(out2[: len(out1)]) == json.dumps(out1) + + +@pytest.mark.asyncio +async def test_implicit_cache_frozen_messages_count_freezes_exact_prefix(): + """An explicit count freezes only that many leading rows; later rows are + still handed to the compression service and come back rewritten.""" + guardrail = _make_guardrail( + implicit_cache_frozen_messages=2, + llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"), + ) + turn1 = _agentic_request(2) + turn2 = _agentic_request(3) + + results, posts = await _apply_two_turns(guardrail, turn1, turn2, model="gpt") + + assert posts + out1 = results[0]["structured_messages"] + out2 = results[1]["structured_messages"] + assert out1[:2] == turn1[:2] + assert out2[:2] == turn2[:2] + assert json.dumps(out2[:2]) == json.dumps(out1[:2]) + assert out1[3]["content"] == "COMPRESSED" + assert [m["role"] for m in posts[0]] == ["tool", "user"] + + +@pytest.mark.asyncio +async def test_implicit_cache_protection_skipped_when_request_carries_cache_control(): + """A request that pins its own breakpoints keeps the existing cache_control + protection path; the frozen-prefix default must not take over.""" + guardrail = _make_guardrail( + implicit_cache_frozen_messages=None, + llm_router_getter=lambda: _StubRouter("openai/gpt-5.6"), + ) + turn1 = _agentic_request(2) + turn1[1]["cache_control"] = {"type": "ephemeral"} + + request_data = {"model": "gpt", "messages": json.loads(json.dumps(turn1))} + posts = [] + + def _record(**kwargs): + posts.append(kwargs["json"]["messages"]) + return _rewrite_every_row(**kwargs) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record): + result = await guardrail.apply_guardrail( + inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(turn1))), + request_data=request_data, + input_type="request", + ) + + assert posts + assert result["structured_messages"][0] == turn1[0] + assert result["structured_messages"][1] == turn1[1] + assert result["structured_messages"][-1] == turn1[-1] + assert result["structured_messages"][3]["content"] == "COMPRESSED" + + +@pytest.mark.asyncio +async def test_model_without_prompt_caching_keeps_compressing_history(): + guardrail = _make_guardrail( + implicit_cache_frozen_messages=None, + llm_router_getter=lambda: _StubRouter("openai/o1-mini"), + ) + messages = _agentic_request(2) + + posts = [] + + def _record(**kwargs): + posts.append(kwargs["json"]["messages"]) + return _rewrite_every_row(**kwargs) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_record): + result = await guardrail.apply_guardrail( + inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))), + request_data={"model": "o1", "messages": json.loads(json.dumps(messages))}, + input_type="request", + ) + + assert posts + assert result["structured_messages"][3]["content"] == "COMPRESSED" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7aa34c5752c..6ea7419144d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32132,6 +32132,11 @@ export interface components { * @description Enable hallucination detection to detect factual inaccuracies. */ hallucinations_check?: boolean | null; + /** + * Implicit Cache Frozen Messages + * @description Leading message count held back from compression when the target model caches on an exact request prefix (no cache_control markers in the request). Unset freezes the whole conversation so every turn stays an append-only prefix of the next; an explicit count trades cache hits above that row for compression; 0 disables the protection. + */ + implicit_cache_frozen_messages?: number | null; /** * Include Evidence * @description Include detailed evidence payloads in responses (sets `plr_evidence` header).