This commit is contained in:
clarence 2026-10-26 17:51:35 -07:00 • committed by GitHub
commit cef092fe65
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 460 additions and 48 deletions

View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
import httpx
from typing_extensions import TypeIs # noqa: TID251 # narrows the metadata container without copying it
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -118,11 +119,33 @@ def is_guardrail_intervention(e: Exception) -> bool:
return is_fastapi_http_exception(e, _GUARDRAIL_BLOCK_STATUS_CODES)
def _user_api_key_auth_from_request(request_data: Mapping[str, object]) -> "UserAPIKeyAuth":
def _is_str_object_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: JSON metadata keys are str
return isinstance(value, Mapping)
def _stamped_metadata(request_data: Mapping[str, object]) -> Mapping[str, object]:
"""The metadata container the proxy stamped the caller's identity into.
The unified guardrail path adds a `litellm_metadata` container built from the
auth object before `apply_guardrail` runs, so on chat routes that container
exists without the stamped fields while `metadata` carries them.
"""
buckets: Final = tuple(
bucket
for bucket in (request_data.get("litellm_metadata"), request_data.get("metadata"))
if _is_str_object_mapping(bucket)
)
stamped: Final = next((bucket for bucket in buckets if "user_api_key_hash" in bucket), None)
if stamped is not None:
return stamped
fallback: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
return fallback if _is_str_object_mapping(fallback) else {}
def user_api_key_auth_from_request_data(request_data: Mapping[str, object]) -> "UserAPIKeyAuth":
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
stamped: Final[Mapping[str, object]] = metadata if isinstance(metadata, dict) else {}
stamped: Final = _stamped_metadata(request_data)
def stamped_str(field: str) -> str | None:
value: Final = stamped.get(field)
@ -915,7 +938,7 @@ class CustomGuardrail(CustomLogger):
if target is not self:
kwargs["guardrail_to_apply"] = self
result: Final = await target.async_pre_call_hook(
user_api_key_dict=_user_api_key_auth_from_request(kwargs),
user_api_key_dict=user_api_key_auth_from_request_data(kwargs),
cache=dc,
data=kwargs,
call_type="completion" if call_type == CallTypes.completion else "acompletion",
@ -946,7 +969,7 @@ class CustomGuardrail(CustomLogger):
}
try:
result: Final = await target.async_pre_call_hook(
user_api_key_dict=_user_api_key_auth_from_request(scan_request),
user_api_key_dict=user_api_key_auth_from_request_data(scan_request),
cache=dc,
data=scan_request,
call_type="acompletion",
@ -996,7 +1019,7 @@ class CustomGuardrail(CustomLogger):
if target is not self:
request_data["guardrail_to_apply"] = self # rebind-ok: dispatch consumes this key
result: Final = await target.async_post_call_success_hook(
user_api_key_dict=_user_api_key_auth_from_request(request_data),
user_api_key_dict=user_api_key_auth_from_request_data(request_data),
data=request_data,
response=response,
)

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import json
import math
import re
@ -21,6 +22,7 @@ from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
user_api_key_auth_from_request_data,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
get_attribute_or_key,
@ -28,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
group_tool_exchanges,
has_tool_with_name,
)
from litellm.litellm_core_utils.provider_affinity import get_stable_session_id
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
httpxSpecialProvider,
@ -56,6 +59,8 @@ if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
BYPASS_HEADER: Final = "x-headroom-bypass"
AFFINITY_HEADER: Final = "x-headroom-affinity"
_AFFINITY_KEY_LENGTH: Final = 32
_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
(CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
)
@ -87,6 +92,27 @@ def _is_object_list(value: object) -> TypeGuard[list[object]]: # guard-ok: isin
return isinstance(value, list)
def _affinity_key(request_data: Mapping[str, object]) -> str | None:
"""Opaque routing value for related Headroom calls, or None.
Both halves must be present: without a session id there is nothing to keep
together, and without a key the proxy authenticated the same session id could
belong to different tenants. Neither raw value leaves the proxy.
"""
session_id: Final = get_stable_session_id(request_data)
key_hash: Final = user_api_key_auth_from_request_data(request_data).api_key
if not session_id or not key_hash:
return None
return hashlib.sha256(f"{key_hash}|{session_id}".encode()).hexdigest()[:_AFFINITY_KEY_LENGTH]
@dataclass(frozen=True, slots=True)
class _IssuedHashes:
hashes: frozenset[str]
expires_at: float
affinity: str | None
def _flatten_messages_for_compression(messages: list[dict[str, object]]) -> list[dict[str, object]]:
"""Collapse all-text list-of-parts content to plain strings for /v1/compress.
@ -521,7 +547,7 @@ class HeadroomGuardrail(CustomGuardrail):
self.async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback,
)
self._issued_hashes_by_call_id: dict[str, tuple[frozenset[str], float]] = {}
self._issued_hashes_by_call_id: dict[str, _IssuedHashes] = {}
super().__init__( # pyright: ignore[reportUnknownMemberType]
guardrail_name=guardrail_name,
event_hook=event_hook,
@ -540,10 +566,12 @@ class HeadroomGuardrail(CustomGuardrail):
value: Final = headers.get(BYPASS_HEADER)
return str(value).lower() == "true"
def _request_headers(self) -> dict[str, str]:
def _request_headers(self, affinity: str | None = None) -> dict[str, str]:
headers: Final[dict[str, str]] = {"Content-Type": "application/json"}
if self.headroom_api_key:
headers["Authorization"] = f"Bearer {self.headroom_api_key}"
if affinity:
headers[AFFINITY_HEADER] = affinity
return headers
@staticmethod
@ -572,9 +600,7 @@ class HeadroomGuardrail(CustomGuardrail):
def _prune_expired_hashes(self) -> None:
now: Final = time.monotonic()
self._issued_hashes_by_call_id = {
call_id: (hashes, expiry)
for call_id, (hashes, expiry) in self._issued_hashes_by_call_id.items()
if expiry > now
call_id: issued for call_id, issued in self._issued_hashes_by_call_id.items() if issued.expires_at > now
}
def _handle_compress_failure(
@ -596,6 +622,7 @@ class HeadroomGuardrail(CustomGuardrail):
self,
messages: list[dict[str, object]],
model: str | None,
affinity: str | None = None,
) -> _CompressResult:
payload: Final[dict[str, object]] = {"messages": messages}
if model:
@ -605,7 +632,7 @@ class HeadroomGuardrail(CustomGuardrail):
raw_response: HttpxResponse = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.post is untyped
url=f"{self.headroom_api_base}/v1/compress",
json=payload,
headers=self._request_headers(),
headers=self._request_headers(affinity),
timeout=self.timeout,
)
except httpx.HTTPStatusError as e:
@ -734,7 +761,7 @@ class HeadroomGuardrail(CustomGuardrail):
stats["tokens_saved"] = tokens_before - tokens_after
return _CompressResult(filtered, True, stats, _read_ccr_hashes(body))
async def _call_retrieve(self, hash_value: str, query: str | None = None) -> str:
async def _call_retrieve(self, hash_value: str, query: str | None = None, affinity: str | None = None) -> str:
params: Final[dict[str, str]] = {}
if query:
params["query"] = query
@ -743,7 +770,7 @@ class HeadroomGuardrail(CustomGuardrail):
raw_response: HttpxResponse = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.get is untyped
url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}",
params=params,
headers=self._request_headers(),
headers=self._request_headers(affinity),
timeout=self.timeout,
)
except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError, litellm.Timeout) as e:
@ -811,18 +838,20 @@ class HeadroomGuardrail(CustomGuardrail):
# user message, so it is withheld here at some cost to history ranking.
# request_data is a bare dict on the base signature; narrow it before
# reading the untranslated messages so long tool names can be recovered.
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
narrowed_request_data: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data)
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(narrowed_request_data.get("messages"))
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
if not compressible:
return inputs
model: Final = self.headroom_model or request_data.get("model")
affinity: Final = _affinity_key(narrowed_request_data)
start_time: Final = time.time()
result: Final = await self._call_compress(
messages=_flatten_messages_for_compression(compressible),
model=model if isinstance(model, str) else None,
affinity=affinity,
)
end_time: Final = time.time()
@ -873,7 +902,11 @@ class HeadroomGuardrail(CustomGuardrail):
if not call_id:
call_id = str(uuid.uuid4())
request_data["litellm_call_id"] = call_id
self._issued_hashes_by_call_id[call_id] = (frozenset(hashes), time.monotonic() + _HASH_CACHE_TTL_SECONDS)
self._issued_hashes_by_call_id[call_id] = _IssuedHashes(
hashes=frozenset(hashes),
expires_at=time.monotonic() + _HASH_CACHE_TTL_SECONDS,
affinity=affinity,
)
existing_tools: Final = inputs.get("tools")
retrieve_tool: Final = _build_headroom_retrieve_tool()
@ -936,7 +969,9 @@ class HeadroomGuardrail(CustomGuardrail):
self._prune_expired_hashes()
call_id: Final = _resolve_call_id(logging_obj, kwargs)
valid_hashes = self._issued_hashes_by_call_id.get(call_id, (frozenset(), 0.0))[0] if call_id else frozenset()
issued: Final = self._issued_hashes_by_call_id.get(call_id) if call_id else None
valid_hashes: Final = issued.hashes if issued else frozenset()
affinity: Final = issued.affinity if issued else None
retrieved: Final[list[tuple[dict[str, object], str]]] = []
for tc in tool_calls:
@ -959,6 +994,7 @@ class HeadroomGuardrail(CustomGuardrail):
content = await self._call_retrieve(
hash_value=hash_value,
query=str(query) if query else None,
affinity=affinity,
)
verbose_proxy_logger.debug("Headroom CCR: retrieved hash=%s (%d chars)", hash_value, len(content))
retrieved.append((tc, content))

View file

@ -0,0 +1,129 @@
"""The Headroom guardrail tags its outbound calls with an opaque session affinity, through a real proxy.
`x-headroom-affinity` is derived from the validated virtual key and the client's own session id and
nothing else, so the compression service can pin related calls together without ever seeing either
raw value. The value must repeat for the same key and session, change when the key changes, ignore
identity fields a client forges into the request body, and be absent when the client sent no session
id. Everything is asserted on the requests the owned compression sink recorded.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
import yaml
from integration._support.client import Gateway, object_value
from integration._support.openai_wire import chat_reply
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
AFFINITY_HEADER: Final = "x-headroom-affinity"
SESSION_HEADER: Final = "x-litellm-session-id"
COMPRESS_PATH: Final = "/v1/compress"
AFFINITY_SHAPE: Final = re.compile(r"[0-9a-f]{32}")
SESSION_ID: Final = "3f1c9b7e2d4a4c6e8b0f1a2c3d4e5f60"
FORGED_IDENTITY: Final[Mapping[str, JsonValue]] = MappingProxyType(
{"user_api_key_team_id": "victim", "user_api_key_hash": "forged"}
)
MESSAGES: Final[tuple[dict[str, JsonValue], ...]] = (
{"role": "system", "content": "You are a terse assistant."},
{"role": "user", "content": "Summarize what session affinity buys a compression service."},
{"role": "assistant", "content": "It keeps related calls on one node so cached context is reused."},
{"role": "user", "content": "And what identifies the session to that service?"},
)
_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
_JSON: Final = TypeAdapter(dict[str, JsonValue])
def _compress(request: Request) -> Reply:
body: Final = _JSON.validate_json(request.body)
return Reply(body=json.dumps({"messages": body["messages"], "tokens_before": 1000, "tokens_after": 100}).encode())
def _provider(_request: Request) -> Reply:
return chat_reply("chatcmpl-headroom-affinity", "gpt-4o-mini", "ok", stream=False)
@dataclass(frozen=True, slots=True)
class Rig:
owned: OwnedProxy
compress: Wire
chat_model: str
key_a: str
key_b: str
@pytest.fixture(scope="module")
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
directory: Final = tmp_path_factory.mktemp("headroom-affinity")
with wire_server(_compress) as compress, wire_server(_provider) as provider:
config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text()))
path: Final = directory / "headroom-affinity.yaml"
path.write_text(
yaml.safe_dump(
{
**config,
"litellm_settings": {**object_value(config["litellm_settings"]), "cache": False},
"guardrails": [
{
"guardrail_name": "headroom",
"litellm_params": {
"guardrail": "headroom",
"mode": "pre_call",
"default_on": True,
"api_base": compress.url,
"api_key": "synthetic-headroom-key",
},
}
],
}
)
)
with (
owned_proxy_process(session_gateway, directory, {}, config=path) as owned,
owned.gateway.scenario() as scenario,
):
chat_model: Final = scenario.model(api_base=provider.url + "/v1")
yield Rig(owned, compress, chat_model, scenario.key(), scenario.key())
def _affinity_sent(
rig: Rig, key: str, *, session: str | None, body: Mapping[str, JsonValue] = MappingProxyType({})
) -> str | None:
rig.compress.drain()
response: Final = rig.owned.gateway.request(
"POST",
"/v1/chat/completions",
{"model": rig.chat_model, "messages": list(MESSAGES), **body},
key=key,
headers={SESSION_HEADER: session} if session is not None else None,
)
assert response.status_code == 200, response.text
compress_calls: Final = tuple(request for request in rig.compress.drain() if request.target == COMPRESS_PATH)
assert len(compress_calls) == 1, [request.target for request in compress_calls]
return compress_calls[0].headers.get(AFFINITY_HEADER)
def test_affinity_repeats_per_key_and_session_without_exposing_either(rig: Rig) -> None:
first: Final = _affinity_sent(rig, rig.key_a, session=SESSION_ID)
assert first is not None and AFFINITY_SHAPE.fullmatch(first), first
assert _affinity_sent(rig, rig.key_a, session=SESSION_ID) == first
assert first not in SESSION_ID
assert first not in sha256(rig.key_a.encode()).hexdigest()
assert _affinity_sent(rig, rig.key_a, session=SESSION_ID, body=FORGED_IDENTITY) == first
other_key: Final = _affinity_sent(rig, rig.key_b, session=SESSION_ID)
assert other_key is not None and AFFINITY_SHAPE.fullmatch(other_key), other_key
assert other_key != first
def test_no_client_session_id_sends_no_affinity(rig: Rig) -> None:
assert _affinity_sent(rig, rig.key_a, session=None) is None

View file

@ -15,6 +15,7 @@ from litellm.integrations.custom_guardrail import (
_request_is_streaming,
guardrail_request_data_with_streaming,
log_guardrail_information,
user_api_key_auth_from_request_data,
without_server_streaming_classification,
)
from litellm.litellm_core_utils.litellm_logging import Logging
@ -3786,3 +3787,41 @@ def test_default_on_guardrail() -> None:
)
is True
)
_STAMPED_IDENTITY: Final = {
"user_api_key_hash": "hash-a",
"user_api_key_team_id": "team-a",
"user_api_key_user_id": "user-a",
}
@pytest.mark.parametrize(
"request_data",
[
pytest.param({"model": "m", "metadata": _STAMPED_IDENTITY}, id="chat route"),
pytest.param(
{"model": "m", "metadata": _STAMPED_IDENTITY, "litellm_metadata": {"user_api_key_token": "hash-a"}},
id="chat route after the unified guardrail adds litellm_metadata",
),
pytest.param(
{"model": "m", "litellm_metadata": _STAMPED_IDENTITY, "metadata": {"user_api_key_hash": "forged"}},
id="messages route with caller metadata",
),
],
)
def test_user_api_key_auth_from_request_data_reads_the_stamped_container(request_data: dict[str, object]) -> None:
auth: Final = user_api_key_auth_from_request_data(request_data)
assert (auth.api_key, auth.team_id, auth.user_id) == ("hash-a", "team-a", "user-a")
def test_user_api_key_auth_from_request_data_without_a_stamped_hash_keeps_the_route_container() -> None:
auth: Final = user_api_key_auth_from_request_data(
{
"litellm_metadata": {"user_api_key_user_id": "route-user"},
"metadata": {"user_api_key_user_id": "other-user"},
}
)
assert (auth.api_key, auth.user_id) == (None, "route-user")

View file

@ -19,10 +19,17 @@ Tests cover:
- CCR: async_build_agentic_loop_plan calls retrieve endpoint and builds follow-up messages
- CCR: streaming /chat/completions is converted to a non-streaming call so the agentic
loop resolves the retrieve tool call, then fake-streamed back to the client
- session affinity: an explicit session id plus the key hash the proxy stamped
produce a stable x-headroom-affinity header on /v1/compress, /v1/retrieve replays
the value stored with the issued hashes, and the header is omitted whenever
either half is missing
"""
import copy
import json
import re
import time
from collections.abc import Mapping, Sequence
from typing import Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
@ -32,11 +39,12 @@ import respx
from fastapi import HTTPException
import litellm
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
HeadroomGuardrail,
has_headroom_retrieve_tool,
AFFINITY_HEADER,
HEADROOM_RETRIEVE_TOOL_NAME,
HeadroomGuardrail,
_IssuedHashes,
has_headroom_retrieve_tool,
)
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
@ -508,9 +516,10 @@ async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages(
tool_id="call_abc123",
)
messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
guardrail._issued_hashes_by_call_id["call-1"] = (
frozenset({"b573993006976af767214fac"}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
hashes=frozenset({"b573993006976af767214fac"}),
expires_at=time.monotonic() + 999,
affinity=None,
)
with patch.object(
@ -574,9 +583,10 @@ async def test_async_build_agentic_loop_plan_handles_retrieve_404(
"content": "Retrieve more: hash=deadbeef000000000000dead",
}
]
guardrail._issued_hashes_by_call_id["call-1"] = (
frozenset({"deadbeef000000000000dead"}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
hashes=frozenset({"deadbeef000000000000dead"}),
expires_at=time.monotonic() + 999,
affinity=None,
)
with patch.object(
@ -658,9 +668,10 @@ async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_c
request just because the second request echoes that hash-shaped string
back in its own messages -- retrieval must be scoped per litellm_call_id,
not derived by re-scanning attacker-controlled message text."""
guardrail._issued_hashes_by_call_id["call-A"] = (
frozenset({"b573993006976af767214fac"}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["call-A"] = _IssuedHashes(
hashes=frozenset({"b573993006976af767214fac"}),
expires_at=time.monotonic() + 999,
affinity=None,
)
tool_calls = [
@ -734,9 +745,10 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_
}
]
messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
guardrail._issued_hashes_by_call_id["call-1"] = (
frozenset({"b573993006976af767214fac"}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
hashes=frozenset({"b573993006976af767214fac"}),
expires_at=time.monotonic() + 999,
affinity=None,
)
with patch.object(
@ -808,9 +820,10 @@ async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messag
}
]
messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
guardrail._issued_hashes_by_call_id["call-1"] = (
frozenset({"b573993006976af767214fac"}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
hashes=frozenset({"b573993006976af767214fac"}),
expires_at=time.monotonic() + 999,
affinity=None,
)
with patch.object(
@ -900,8 +913,8 @@ async def test_service_declared_ccr_hashes_drive_injection_and_validation(guardr
)
assert has_headroom_retrieve_tool(result.get("tools") or [])
((issued, _expiry),) = guardrail._issued_hashes_by_call_id.values()
assert issued == frozenset({"98ca69107318", "b573993006976af767214fac"})
(issued,) = guardrail._issued_hashes_by_call_id.values()
assert issued.hashes == frozenset({"98ca69107318", "b573993006976af767214fac"})
@pytest.mark.asyncio
@ -2313,9 +2326,10 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end(
to a client that never declared the tool."""
original_content = "the full uncompressed document"
final_answer = "the document says hello"
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
frozenset({CCR_HASH}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes(
hashes=frozenset({CCR_HASH}),
expires_at=time.monotonic() + 999,
affinity=None,
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
@ -2367,9 +2381,10 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end(
instead of streaming a headroom_retrieve function_call to the client."""
original_content = "the full uncompressed document"
final_answer = "the document says hello"
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
frozenset({CCR_HASH}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes(
hashes=frozenset({CCR_HASH}),
expires_at=time.monotonic() + 999,
affinity=None,
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
@ -2423,9 +2438,10 @@ def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end(
completed response object."""
original_content = "the full uncompressed document"
final_answer = "the document says hello"
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
frozenset({CCR_HASH}),
time.monotonic() + 999,
guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes(
hashes=frozenset({CCR_HASH}),
expires_at=time.monotonic() + 999,
affinity=None,
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
@ -2849,7 +2865,9 @@ def _anthropic_response_with_text_and_tool_call() -> dict:
async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list):
guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({"h" * 24}), time.monotonic() + 60)
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
hashes=frozenset({"h" * 24}), expires_at=time.monotonic() + 60, affinity=None
)
logging_obj = MagicMock()
logging_obj.litellm_call_id = "call-1"
logging_obj.model_call_details = {}
@ -3075,3 +3093,170 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float):
assert guardrail.timeout.read == 60.0
assert guardrail.timeout.connect == 5.0
CHAT_REQUEST: Final[Mapping[str, object]] = {
"model": "gpt-4o",
"litellm_session_id": "sess-abc",
"metadata": {"user_api_key_hash": "hash-a"},
}
MESSAGES_REQUEST: Final[Mapping[str, object]] = {
"model": "gpt-4o",
"litellm_session_id": "sess-abc",
"litellm_metadata": {"user_api_key_hash": "hash-a"},
}
FORGED_IDENTITY: Final[Mapping[str, object]] = {"user_api_key_hash": "forged", "user_api_key_team_id": "victim"}
def _affinity_of(call: respx.models.Call) -> str | None:
value: Final = call.request.headers.get(AFFINITY_HEADER)
return str(value) if value else None
def _captured_affinity(route: respx.Route) -> str | None:
return _affinity_of(route.calls[-1])
def _mock_compress(respx_mock: respx.MockRouter, ccr_hashes: Sequence[str] = ()) -> respx.Route:
messages: Final = COMPRESSED_MESSAGES_WITH_HASH if ccr_hashes else COMPRESSED_MESSAGES
return respx_mock.post(f"{FAKE_API_BASE}/v1/compress").mock(
return_value=httpx.Response(
200,
json={"messages": messages, "tokens_before": 1000, "tokens_after": 100, "ccr_hashes": list(ccr_hashes)},
)
)
async def _compress_with(guardrail: HeadroomGuardrail, request_data: Mapping[str, object]) -> None:
await guardrail.apply_guardrail(
inputs=GenericGuardrailAPIInputs(texts=["A" * 5000], structured_messages=ORIGINAL_MESSAGES),
request_data=copy.deepcopy(dict(request_data)),
input_type="request",
)
@pytest.mark.asyncio
async def test_compress_sends_stable_affinity_for_one_conversation(
httpx_transport: None,
guardrail: HeadroomGuardrail,
respx_mock: respx.MockRouter,
) -> None:
route: Final = _mock_compress(respx_mock)
await _compress_with(guardrail, CHAT_REQUEST)
await _compress_with(guardrail, CHAT_REQUEST)
first, second = tuple(_affinity_of(call) for call in route.calls)
assert first is not None
assert second == first
assert re.fullmatch(r"[0-9a-f]{32}", first)
assert "sess-abc" not in first
assert "hash-a" not in first
@pytest.mark.asyncio
async def test_affinity_differs_per_key_for_the_same_session_id(
httpx_transport: None,
guardrail: HeadroomGuardrail,
respx_mock: respx.MockRouter,
) -> None:
route: Final = _mock_compress(respx_mock)
await _compress_with(guardrail, CHAT_REQUEST)
await _compress_with(guardrail, {**CHAT_REQUEST, "metadata": {"user_api_key_hash": "hash-b"}})
key_a, key_b = tuple(_affinity_of(call) for call in route.calls)
assert key_a is not None
assert key_b is not None
assert key_b != key_a
@pytest.mark.asyncio
@pytest.mark.parametrize(
"request_data",
[
{"model": "gpt-4o", "metadata": {"user_api_key_hash": "hash-a"}},
{**CHAT_REQUEST, "metadata": {"litellm_session_id_generated": True, "user_api_key_hash": "hash-a"}},
{**MESSAGES_REQUEST, "litellm_metadata": {"litellm_session_id_generated": True, "user_api_key_hash": "hash-a"}},
{"model": "gpt-4o", "litellm_session_id": "sess-abc"},
],
ids=["no_session_id", "generated_session_id_chat", "generated_session_id_messages", "no_stamped_identity"],
)
async def test_affinity_omitted_when_either_half_is_missing(
httpx_transport: None,
guardrail: HeadroomGuardrail,
respx_mock: respx.MockRouter,
request_data: Mapping[str, object],
) -> None:
route: Final = _mock_compress(respx_mock)
await _compress_with(guardrail, request_data)
assert _captured_affinity(route) is None
@pytest.mark.asyncio
async def test_affinity_ignores_identity_a_client_puts_in_the_body(
httpx_transport: None,
guardrail: HeadroomGuardrail,
respx_mock: respx.MockRouter,
) -> None:
route: Final = _mock_compress(respx_mock)
await _compress_with(guardrail, CHAT_REQUEST)
await _compress_with(guardrail, {**CHAT_REQUEST, **FORGED_IDENTITY})
await _compress_with(guardrail, {"model": "gpt-4o", "litellm_session_id": "sess-abc", **FORGED_IDENTITY})
stamped, forged_beside_stamped, forged_alone = tuple(_affinity_of(call) for call in route.calls)
assert stamped is not None
assert forged_beside_stamped == stamped
assert forged_alone is None
@pytest.mark.asyncio
async def test_affinity_reads_identity_from_the_container_the_route_stamps(
httpx_transport: None,
guardrail: HeadroomGuardrail,
respx_mock: respx.MockRouter,
) -> None:
route: Final = _mock_compress(respx_mock)
await _compress_with(guardrail, CHAT_REQUEST)
await _compress_with(guardrail, MESSAGES_REQUEST)
await _compress_with(guardrail, {**CHAT_REQUEST, "litellm_metadata": {"user_api_key_token": "hash-a"}})
chat, messages, chat_through_unified_guardrail = tuple(_affinity_of(call) for call in route.calls)
assert chat is not None
assert messages == chat
assert chat_through_unified_guardrail == chat
@pytest.mark.asyncio
async def test_retrieve_replays_the_affinity_the_compress_call_sent(
httpx_transport: None,
guardrail: HeadroomGuardrail,
respx_mock: respx.MockRouter,
) -> None:
compress: Final = _mock_compress(respx_mock, ccr_hashes=(CCR_HASH,))
retrieve: Final = respx_mock.get(f"{FAKE_API_BASE}/v1/retrieve/{CCR_HASH}").mock(
return_value=httpx.Response(200, json={"original_content": "restored"})
)
await _compress_with(guardrail, {**CHAT_REQUEST, "litellm_call_id": "call-1"})
compress_affinity: Final = _captured_affinity(compress)
assert compress_affinity is not None
await guardrail.async_build_agentic_loop_plan(
tools={"tool_calls": [{"arguments": {"hash": CCR_HASH}}]},
model="gpt-4o",
messages=list(ORIGINAL_MESSAGES),
response={},
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={},
logging_obj=None,
stream=False,
kwargs={"litellm_call_id": "call-1"},
)
assert retrieve.called
assert _captured_affinity(retrieve) == compress_affinity