diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 9e2dc902d55..5cccff21505 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -9,6 +9,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args import httpx +from typing_extensions import TypeIs # noqa: TID251 # narrows the metadata container without copying it from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -118,11 +119,33 @@ def is_guardrail_intervention(e: Exception) -> bool: return is_fastapi_http_exception(e, _GUARDRAIL_BLOCK_STATUS_CODES) -def _user_api_key_auth_from_request(request_data: Mapping[str, object]) -> "UserAPIKeyAuth": +def _is_str_object_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: JSON metadata keys are str + return isinstance(value, Mapping) + + +def _stamped_metadata(request_data: Mapping[str, object]) -> Mapping[str, object]: + """The metadata container the proxy stamped the caller's identity into. + + The unified guardrail path adds a `litellm_metadata` container built from the + auth object before `apply_guardrail` runs, so on chat routes that container + exists without the stamped fields while `metadata` carries them. + """ + buckets: Final = tuple( + bucket + for bucket in (request_data.get("litellm_metadata"), request_data.get("metadata")) + if _is_str_object_mapping(bucket) + ) + stamped: Final = next((bucket for bucket in buckets if "user_api_key_hash" in bucket), None) + if stamped is not None: + return stamped + fallback: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data)) + return fallback if _is_str_object_mapping(fallback) else {} + + +def user_api_key_auth_from_request_data(request_data: Mapping[str, object]) -> "UserAPIKeyAuth": from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth - metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data)) - stamped: Final[Mapping[str, object]] = metadata if isinstance(metadata, dict) else {} + stamped: Final = _stamped_metadata(request_data) def stamped_str(field: str) -> str | None: value: Final = stamped.get(field) @@ -915,7 +938,7 @@ class CustomGuardrail(CustomLogger): if target is not self: kwargs["guardrail_to_apply"] = self result: Final = await target.async_pre_call_hook( - user_api_key_dict=_user_api_key_auth_from_request(kwargs), + user_api_key_dict=user_api_key_auth_from_request_data(kwargs), cache=dc, data=kwargs, call_type="completion" if call_type == CallTypes.completion else "acompletion", @@ -946,7 +969,7 @@ class CustomGuardrail(CustomLogger): } try: result: Final = await target.async_pre_call_hook( - user_api_key_dict=_user_api_key_auth_from_request(scan_request), + user_api_key_dict=user_api_key_auth_from_request_data(scan_request), cache=dc, data=scan_request, call_type="acompletion", @@ -996,7 +1019,7 @@ class CustomGuardrail(CustomLogger): if target is not self: request_data["guardrail_to_apply"] = self # rebind-ok: dispatch consumes this key result: Final = await target.async_post_call_success_hook( - user_api_key_dict=_user_api_key_auth_from_request(request_data), + user_api_key_dict=user_api_key_auth_from_request_data(request_data), data=request_data, response=response, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index dac802a1ddc..c5fd118ca5a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import math import re @@ -21,6 +22,7 @@ from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, + user_api_key_auth_from_request_data, ) from litellm.litellm_core_utils.prompt_templates.factory import ( get_attribute_or_key, @@ -28,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( group_tool_exchanges, has_tool_with_name, ) +from litellm.litellm_core_utils.provider_affinity import get_stable_session_id from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] httpxSpecialProvider, @@ -56,6 +59,8 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel BYPASS_HEADER: Final = "x-headroom-bypass" +AFFINITY_HEADER: Final = "x-headroom-affinity" +_AFFINITY_KEY_LENGTH: Final = 32 _STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset( (CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses) ) @@ -87,6 +92,27 @@ def _is_object_list(value: object) -> TypeGuard[list[object]]: # guard-ok: isin return isinstance(value, list) +def _affinity_key(request_data: Mapping[str, object]) -> str | None: + """Opaque routing value for related Headroom calls, or None. + + Both halves must be present: without a session id there is nothing to keep + together, and without a key the proxy authenticated the same session id could + belong to different tenants. Neither raw value leaves the proxy. + """ + session_id: Final = get_stable_session_id(request_data) + key_hash: Final = user_api_key_auth_from_request_data(request_data).api_key + if not session_id or not key_hash: + return None + return hashlib.sha256(f"{key_hash}|{session_id}".encode()).hexdigest()[:_AFFINITY_KEY_LENGTH] + + +@dataclass(frozen=True, slots=True) +class _IssuedHashes: + hashes: frozenset[str] + expires_at: float + affinity: str | None + + def _flatten_messages_for_compression(messages: list[dict[str, object]]) -> list[dict[str, object]]: """Collapse all-text list-of-parts content to plain strings for /v1/compress. @@ -521,7 +547,7 @@ class HeadroomGuardrail(CustomGuardrail): self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) - self._issued_hashes_by_call_id: dict[str, tuple[frozenset[str], float]] = {} + self._issued_hashes_by_call_id: dict[str, _IssuedHashes] = {} super().__init__( # pyright: ignore[reportUnknownMemberType] guardrail_name=guardrail_name, event_hook=event_hook, @@ -540,10 +566,12 @@ class HeadroomGuardrail(CustomGuardrail): value: Final = headers.get(BYPASS_HEADER) return str(value).lower() == "true" - def _request_headers(self) -> dict[str, str]: + def _request_headers(self, affinity: str | None = None) -> dict[str, str]: headers: Final[dict[str, str]] = {"Content-Type": "application/json"} if self.headroom_api_key: headers["Authorization"] = f"Bearer {self.headroom_api_key}" + if affinity: + headers[AFFINITY_HEADER] = affinity return headers @staticmethod @@ -572,9 +600,7 @@ class HeadroomGuardrail(CustomGuardrail): def _prune_expired_hashes(self) -> None: now: Final = time.monotonic() self._issued_hashes_by_call_id = { - call_id: (hashes, expiry) - for call_id, (hashes, expiry) in self._issued_hashes_by_call_id.items() - if expiry > now + call_id: issued for call_id, issued in self._issued_hashes_by_call_id.items() if issued.expires_at > now } def _handle_compress_failure( @@ -596,6 +622,7 @@ class HeadroomGuardrail(CustomGuardrail): self, messages: list[dict[str, object]], model: str | None, + affinity: str | None = None, ) -> _CompressResult: payload: Final[dict[str, object]] = {"messages": messages} if model: @@ -605,7 +632,7 @@ class HeadroomGuardrail(CustomGuardrail): raw_response: HttpxResponse = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.post is untyped url=f"{self.headroom_api_base}/v1/compress", json=payload, - headers=self._request_headers(), + headers=self._request_headers(affinity), timeout=self.timeout, ) except httpx.HTTPStatusError as e: @@ -734,7 +761,7 @@ class HeadroomGuardrail(CustomGuardrail): stats["tokens_saved"] = tokens_before - tokens_after return _CompressResult(filtered, True, stats, _read_ccr_hashes(body)) - async def _call_retrieve(self, hash_value: str, query: str | None = None) -> str: + async def _call_retrieve(self, hash_value: str, query: str | None = None, affinity: str | None = None) -> str: params: Final[dict[str, str]] = {} if query: params["query"] = query @@ -743,7 +770,7 @@ class HeadroomGuardrail(CustomGuardrail): raw_response: HttpxResponse = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.get is untyped url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}", params=params, - headers=self._request_headers(), + headers=self._request_headers(affinity), timeout=self.timeout, ) except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError, litellm.Timeout) as e: @@ -811,18 +838,20 @@ class HeadroomGuardrail(CustomGuardrail): # user message, so it is withheld here at some cost to history ranking. # request_data is a bare dict on the base signature; narrow it before # reading the untranslated messages so long tool names can be recovered. - raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") - raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) + narrowed_request_data: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data) + raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(narrowed_request_data.get("messages")) protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] if not compressible: return inputs model: Final = self.headroom_model or request_data.get("model") + affinity: Final = _affinity_key(narrowed_request_data) start_time: Final = time.time() result: Final = await self._call_compress( messages=_flatten_messages_for_compression(compressible), model=model if isinstance(model, str) else None, + affinity=affinity, ) end_time: Final = time.time() @@ -873,7 +902,11 @@ class HeadroomGuardrail(CustomGuardrail): if not call_id: call_id = str(uuid.uuid4()) request_data["litellm_call_id"] = call_id - self._issued_hashes_by_call_id[call_id] = (frozenset(hashes), time.monotonic() + _HASH_CACHE_TTL_SECONDS) + self._issued_hashes_by_call_id[call_id] = _IssuedHashes( + hashes=frozenset(hashes), + expires_at=time.monotonic() + _HASH_CACHE_TTL_SECONDS, + affinity=affinity, + ) existing_tools: Final = inputs.get("tools") retrieve_tool: Final = _build_headroom_retrieve_tool() @@ -936,7 +969,9 @@ class HeadroomGuardrail(CustomGuardrail): self._prune_expired_hashes() call_id: Final = _resolve_call_id(logging_obj, kwargs) - valid_hashes = self._issued_hashes_by_call_id.get(call_id, (frozenset(), 0.0))[0] if call_id else frozenset() + issued: Final = self._issued_hashes_by_call_id.get(call_id) if call_id else None + valid_hashes: Final = issued.hashes if issued else frozenset() + affinity: Final = issued.affinity if issued else None retrieved: Final[list[tuple[dict[str, object], str]]] = [] for tc in tool_calls: @@ -959,6 +994,7 @@ class HeadroomGuardrail(CustomGuardrail): content = await self._call_retrieve( hash_value=hash_value, query=str(query) if query else None, + affinity=affinity, ) verbose_proxy_logger.debug("Headroom CCR: retrieved hash=%s (%d chars)", hash_value, len(content)) retrieved.append((tc, content)) diff --git a/tests/integration/observability/test_headroom_affinity.py b/tests/integration/observability/test_headroom_affinity.py new file mode 100644 index 00000000000..711999f068e --- /dev/null +++ b/tests/integration/observability/test_headroom_affinity.py @@ -0,0 +1,129 @@ +"""The Headroom guardrail tags its outbound calls with an opaque session affinity, through a real proxy. + +`x-headroom-affinity` is derived from the validated virtual key and the client's own session id and +nothing else, so the compression service can pin related calls together without ever seeing either +raw value. The value must repeat for the same key and session, change when the key changes, ignore +identity fields a client forges into the request body, and be absent when the client sent no session +id. Everything is asserted on the requests the owned compression sink recorded. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, object_value +from integration._support.openai_wire import chat_reply +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +AFFINITY_HEADER: Final = "x-headroom-affinity" +SESSION_HEADER: Final = "x-litellm-session-id" +COMPRESS_PATH: Final = "/v1/compress" +AFFINITY_SHAPE: Final = re.compile(r"[0-9a-f]{32}") +SESSION_ID: Final = "3f1c9b7e2d4a4c6e8b0f1a2c3d4e5f60" +FORGED_IDENTITY: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"user_api_key_team_id": "victim", "user_api_key_hash": "forged"} +) +MESSAGES: Final[tuple[dict[str, JsonValue], ...]] = ( + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Summarize what session affinity buys a compression service."}, + {"role": "assistant", "content": "It keeps related calls on one node so cached context is reused."}, + {"role": "user", "content": "And what identifies the session to that service?"}, +) +_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +_JSON: Final = TypeAdapter(dict[str, JsonValue]) + + +def _compress(request: Request) -> Reply: + body: Final = _JSON.validate_json(request.body) + return Reply(body=json.dumps({"messages": body["messages"], "tokens_before": 1000, "tokens_after": 100}).encode()) + + +def _provider(_request: Request) -> Reply: + return chat_reply("chatcmpl-headroom-affinity", "gpt-4o-mini", "ok", stream=False) + + +@dataclass(frozen=True, slots=True) +class Rig: + owned: OwnedProxy + compress: Wire + chat_model: str + key_a: str + key_b: str + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("headroom-affinity") + with wire_server(_compress) as compress, wire_server(_provider) as provider: + config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text())) + path: Final = directory / "headroom-affinity.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "litellm_settings": {**object_value(config["litellm_settings"]), "cache": False}, + "guardrails": [ + { + "guardrail_name": "headroom", + "litellm_params": { + "guardrail": "headroom", + "mode": "pre_call", + "default_on": True, + "api_base": compress.url, + "api_key": "synthetic-headroom-key", + }, + } + ], + } + ) + ) + with ( + owned_proxy_process(session_gateway, directory, {}, config=path) as owned, + owned.gateway.scenario() as scenario, + ): + chat_model: Final = scenario.model(api_base=provider.url + "/v1") + yield Rig(owned, compress, chat_model, scenario.key(), scenario.key()) + + +def _affinity_sent( + rig: Rig, key: str, *, session: str | None, body: Mapping[str, JsonValue] = MappingProxyType({}) +) -> str | None: + rig.compress.drain() + response: Final = rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": rig.chat_model, "messages": list(MESSAGES), **body}, + key=key, + headers={SESSION_HEADER: session} if session is not None else None, + ) + assert response.status_code == 200, response.text + compress_calls: Final = tuple(request for request in rig.compress.drain() if request.target == COMPRESS_PATH) + assert len(compress_calls) == 1, [request.target for request in compress_calls] + return compress_calls[0].headers.get(AFFINITY_HEADER) + + +def test_affinity_repeats_per_key_and_session_without_exposing_either(rig: Rig) -> None: + first: Final = _affinity_sent(rig, rig.key_a, session=SESSION_ID) + assert first is not None and AFFINITY_SHAPE.fullmatch(first), first + assert _affinity_sent(rig, rig.key_a, session=SESSION_ID) == first + assert first not in SESSION_ID + assert first not in sha256(rig.key_a.encode()).hexdigest() + assert _affinity_sent(rig, rig.key_a, session=SESSION_ID, body=FORGED_IDENTITY) == first + other_key: Final = _affinity_sent(rig, rig.key_b, session=SESSION_ID) + assert other_key is not None and AFFINITY_SHAPE.fullmatch(other_key), other_key + assert other_key != first + + +def test_no_client_session_id_sends_no_affinity(rig: Rig) -> None: + assert _affinity_sent(rig, rig.key_a, session=None) is None diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 614ff97553a..1647e0b9f22 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -15,6 +15,7 @@ from litellm.integrations.custom_guardrail import ( _request_is_streaming, guardrail_request_data_with_streaming, log_guardrail_information, + user_api_key_auth_from_request_data, without_server_streaming_classification, ) from litellm.litellm_core_utils.litellm_logging import Logging @@ -3786,3 +3787,41 @@ def test_default_on_guardrail() -> None: ) is True ) + + +_STAMPED_IDENTITY: Final = { + "user_api_key_hash": "hash-a", + "user_api_key_team_id": "team-a", + "user_api_key_user_id": "user-a", +} + + +@pytest.mark.parametrize( + "request_data", + [ + pytest.param({"model": "m", "metadata": _STAMPED_IDENTITY}, id="chat route"), + pytest.param( + {"model": "m", "metadata": _STAMPED_IDENTITY, "litellm_metadata": {"user_api_key_token": "hash-a"}}, + id="chat route after the unified guardrail adds litellm_metadata", + ), + pytest.param( + {"model": "m", "litellm_metadata": _STAMPED_IDENTITY, "metadata": {"user_api_key_hash": "forged"}}, + id="messages route with caller metadata", + ), + ], +) +def test_user_api_key_auth_from_request_data_reads_the_stamped_container(request_data: dict[str, object]) -> None: + auth: Final = user_api_key_auth_from_request_data(request_data) + + assert (auth.api_key, auth.team_id, auth.user_id) == ("hash-a", "team-a", "user-a") + + +def test_user_api_key_auth_from_request_data_without_a_stamped_hash_keeps_the_route_container() -> None: + auth: Final = user_api_key_auth_from_request_data( + { + "litellm_metadata": {"user_api_key_user_id": "route-user"}, + "metadata": {"user_api_key_user_id": "other-user"}, + } + ) + + assert (auth.api_key, auth.user_id) == (None, "route-user") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 5bdcdddf4b7..a733cbd4c31 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -19,10 +19,17 @@ Tests cover: - CCR: async_build_agentic_loop_plan calls retrieve endpoint and builds follow-up messages - CCR: streaming /chat/completions is converted to a non-streaming call so the agentic loop resolves the retrieve tool call, then fake-streamed back to the client +- session affinity: an explicit session id plus the key hash the proxy stamped + produce a stable x-headroom-affinity header on /v1/compress, /v1/retrieve replays + the value stored with the issued hashes, and the header is omitted whenever + either half is missing """ +import copy import json +import re import time +from collections.abc import Mapping, Sequence from typing import Final, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -32,11 +39,12 @@ import respx from fastapi import HTTPException import litellm - from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( - HeadroomGuardrail, - has_headroom_retrieve_tool, + AFFINITY_HEADER, HEADROOM_RETRIEVE_TOOL_NAME, + HeadroomGuardrail, + _IssuedHashes, + has_headroom_retrieve_tool, ) from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, @@ -508,9 +516,10 @@ async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages( tool_id="call_abc123", ) messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes( + hashes=frozenset({"b573993006976af767214fac"}), + expires_at=time.monotonic() + 999, + affinity=None, ) with patch.object( @@ -574,9 +583,10 @@ async def test_async_build_agentic_loop_plan_handles_retrieve_404( "content": "Retrieve more: hash=deadbeef000000000000dead", } ] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"deadbeef000000000000dead"}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes( + hashes=frozenset({"deadbeef000000000000dead"}), + expires_at=time.monotonic() + 999, + affinity=None, ) with patch.object( @@ -658,9 +668,10 @@ async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_c request just because the second request echoes that hash-shaped string back in its own messages -- retrieval must be scoped per litellm_call_id, not derived by re-scanning attacker-controlled message text.""" - guardrail._issued_hashes_by_call_id["call-A"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["call-A"] = _IssuedHashes( + hashes=frozenset({"b573993006976af767214fac"}), + expires_at=time.monotonic() + 999, + affinity=None, ) tool_calls = [ @@ -734,9 +745,10 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_ } ] messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes( + hashes=frozenset({"b573993006976af767214fac"}), + expires_at=time.monotonic() + 999, + affinity=None, ) with patch.object( @@ -808,9 +820,10 @@ async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messag } ] messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes( + hashes=frozenset({"b573993006976af767214fac"}), + expires_at=time.monotonic() + 999, + affinity=None, ) with patch.object( @@ -900,8 +913,8 @@ async def test_service_declared_ccr_hashes_drive_injection_and_validation(guardr ) assert has_headroom_retrieve_tool(result.get("tools") or []) - ((issued, _expiry),) = guardrail._issued_hashes_by_call_id.values() - assert issued == frozenset({"98ca69107318", "b573993006976af767214fac"}) + (issued,) = guardrail._issued_hashes_by_call_id.values() + assert issued.hashes == frozenset({"98ca69107318", "b573993006976af767214fac"}) @pytest.mark.asyncio @@ -2313,9 +2326,10 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end( to a client that never declared the tool.""" original_content = "the full uncompressed document" final_answer = "the document says hello" - guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( - frozenset({CCR_HASH}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes( + hashes=frozenset({CCR_HASH}), + expires_at=time.monotonic() + 999, + affinity=None, ) monkeypatch.setenv("OPENAI_API_KEY", "sk-test") @@ -2367,9 +2381,10 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end( instead of streaming a headroom_retrieve function_call to the client.""" original_content = "the full uncompressed document" final_answer = "the document says hello" - guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( - frozenset({CCR_HASH}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes( + hashes=frozenset({CCR_HASH}), + expires_at=time.monotonic() + 999, + affinity=None, ) monkeypatch.setenv("OPENAI_API_KEY", "sk-test") @@ -2423,9 +2438,10 @@ def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end( completed response object.""" original_content = "the full uncompressed document" final_answer = "the document says hello" - guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( - frozenset({CCR_HASH}), - time.monotonic() + 999, + guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes( + hashes=frozenset({CCR_HASH}), + expires_at=time.monotonic() + 999, + affinity=None, ) monkeypatch.setenv("OPENAI_API_KEY", "sk-test") @@ -2849,7 +2865,9 @@ def _anthropic_response_with_text_and_tool_call() -> dict: async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list): - guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({"h" * 24}), time.monotonic() + 60) + guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes( + hashes=frozenset({"h" * 24}), expires_at=time.monotonic() + 60, affinity=None + ) logging_obj = MagicMock() logging_obj.litellm_call_id = "call-1" logging_obj.model_call_details = {} @@ -3075,3 +3093,170 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float): assert guardrail.timeout.read == 60.0 assert guardrail.timeout.connect == 5.0 + + +CHAT_REQUEST: Final[Mapping[str, object]] = { + "model": "gpt-4o", + "litellm_session_id": "sess-abc", + "metadata": {"user_api_key_hash": "hash-a"}, +} +MESSAGES_REQUEST: Final[Mapping[str, object]] = { + "model": "gpt-4o", + "litellm_session_id": "sess-abc", + "litellm_metadata": {"user_api_key_hash": "hash-a"}, +} +FORGED_IDENTITY: Final[Mapping[str, object]] = {"user_api_key_hash": "forged", "user_api_key_team_id": "victim"} + + +def _affinity_of(call: respx.models.Call) -> str | None: + value: Final = call.request.headers.get(AFFINITY_HEADER) + return str(value) if value else None + + +def _captured_affinity(route: respx.Route) -> str | None: + return _affinity_of(route.calls[-1]) + + +def _mock_compress(respx_mock: respx.MockRouter, ccr_hashes: Sequence[str] = ()) -> respx.Route: + messages: Final = COMPRESSED_MESSAGES_WITH_HASH if ccr_hashes else COMPRESSED_MESSAGES + return respx_mock.post(f"{FAKE_API_BASE}/v1/compress").mock( + return_value=httpx.Response( + 200, + json={"messages": messages, "tokens_before": 1000, "tokens_after": 100, "ccr_hashes": list(ccr_hashes)}, + ) + ) + + +async def _compress_with(guardrail: HeadroomGuardrail, request_data: Mapping[str, object]) -> None: + await guardrail.apply_guardrail( + inputs=GenericGuardrailAPIInputs(texts=["A" * 5000], structured_messages=ORIGINAL_MESSAGES), + request_data=copy.deepcopy(dict(request_data)), + input_type="request", + ) + + +@pytest.mark.asyncio +async def test_compress_sends_stable_affinity_for_one_conversation( + httpx_transport: None, + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, +) -> None: + route: Final = _mock_compress(respx_mock) + + await _compress_with(guardrail, CHAT_REQUEST) + await _compress_with(guardrail, CHAT_REQUEST) + + first, second = tuple(_affinity_of(call) for call in route.calls) + assert first is not None + assert second == first + assert re.fullmatch(r"[0-9a-f]{32}", first) + assert "sess-abc" not in first + assert "hash-a" not in first + + +@pytest.mark.asyncio +async def test_affinity_differs_per_key_for_the_same_session_id( + httpx_transport: None, + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, +) -> None: + route: Final = _mock_compress(respx_mock) + + await _compress_with(guardrail, CHAT_REQUEST) + await _compress_with(guardrail, {**CHAT_REQUEST, "metadata": {"user_api_key_hash": "hash-b"}}) + + key_a, key_b = tuple(_affinity_of(call) for call in route.calls) + assert key_a is not None + assert key_b is not None + assert key_b != key_a + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_data", + [ + {"model": "gpt-4o", "metadata": {"user_api_key_hash": "hash-a"}}, + {**CHAT_REQUEST, "metadata": {"litellm_session_id_generated": True, "user_api_key_hash": "hash-a"}}, + {**MESSAGES_REQUEST, "litellm_metadata": {"litellm_session_id_generated": True, "user_api_key_hash": "hash-a"}}, + {"model": "gpt-4o", "litellm_session_id": "sess-abc"}, + ], + ids=["no_session_id", "generated_session_id_chat", "generated_session_id_messages", "no_stamped_identity"], +) +async def test_affinity_omitted_when_either_half_is_missing( + httpx_transport: None, + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, + request_data: Mapping[str, object], +) -> None: + route: Final = _mock_compress(respx_mock) + + await _compress_with(guardrail, request_data) + + assert _captured_affinity(route) is None + + +@pytest.mark.asyncio +async def test_affinity_ignores_identity_a_client_puts_in_the_body( + httpx_transport: None, + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, +) -> None: + route: Final = _mock_compress(respx_mock) + + await _compress_with(guardrail, CHAT_REQUEST) + await _compress_with(guardrail, {**CHAT_REQUEST, **FORGED_IDENTITY}) + await _compress_with(guardrail, {"model": "gpt-4o", "litellm_session_id": "sess-abc", **FORGED_IDENTITY}) + + stamped, forged_beside_stamped, forged_alone = tuple(_affinity_of(call) for call in route.calls) + assert stamped is not None + assert forged_beside_stamped == stamped + assert forged_alone is None + + +@pytest.mark.asyncio +async def test_affinity_reads_identity_from_the_container_the_route_stamps( + httpx_transport: None, + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, +) -> None: + route: Final = _mock_compress(respx_mock) + + await _compress_with(guardrail, CHAT_REQUEST) + await _compress_with(guardrail, MESSAGES_REQUEST) + await _compress_with(guardrail, {**CHAT_REQUEST, "litellm_metadata": {"user_api_key_token": "hash-a"}}) + + chat, messages, chat_through_unified_guardrail = tuple(_affinity_of(call) for call in route.calls) + assert chat is not None + assert messages == chat + assert chat_through_unified_guardrail == chat + + +@pytest.mark.asyncio +async def test_retrieve_replays_the_affinity_the_compress_call_sent( + httpx_transport: None, + guardrail: HeadroomGuardrail, + respx_mock: respx.MockRouter, +) -> None: + compress: Final = _mock_compress(respx_mock, ccr_hashes=(CCR_HASH,)) + retrieve: Final = respx_mock.get(f"{FAKE_API_BASE}/v1/retrieve/{CCR_HASH}").mock( + return_value=httpx.Response(200, json={"original_content": "restored"}) + ) + + await _compress_with(guardrail, {**CHAT_REQUEST, "litellm_call_id": "call-1"}) + compress_affinity: Final = _captured_affinity(compress) + assert compress_affinity is not None + + await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": [{"arguments": {"hash": CCR_HASH}}]}, + model="gpt-4o", + messages=list(ORIGINAL_MESSAGES), + response={}, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + assert retrieve.called + assert _captured_affinity(retrieve) == compress_affinity