mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge d73759a904 into 100529391a
This commit is contained in:
commit
cef092fe65
5 changed files with 460 additions and 48 deletions
|
|
@ -9,6 +9,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
|
||||
|
||||
import httpx
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows the metadata container without copying it
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -118,11 +119,33 @@ def is_guardrail_intervention(e: Exception) -> bool:
|
|||
return is_fastapi_http_exception(e, _GUARDRAIL_BLOCK_STATUS_CODES)
|
||||
|
||||
|
||||
def _user_api_key_auth_from_request(request_data: Mapping[str, object]) -> "UserAPIKeyAuth":
|
||||
def _is_str_object_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: JSON metadata keys are str
|
||||
return isinstance(value, Mapping)
|
||||
|
||||
|
||||
def _stamped_metadata(request_data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""The metadata container the proxy stamped the caller's identity into.
|
||||
|
||||
The unified guardrail path adds a `litellm_metadata` container built from the
|
||||
auth object before `apply_guardrail` runs, so on chat routes that container
|
||||
exists without the stamped fields while `metadata` carries them.
|
||||
"""
|
||||
buckets: Final = tuple(
|
||||
bucket
|
||||
for bucket in (request_data.get("litellm_metadata"), request_data.get("metadata"))
|
||||
if _is_str_object_mapping(bucket)
|
||||
)
|
||||
stamped: Final = next((bucket for bucket in buckets if "user_api_key_hash" in bucket), None)
|
||||
if stamped is not None:
|
||||
return stamped
|
||||
fallback: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
|
||||
return fallback if _is_str_object_mapping(fallback) else {}
|
||||
|
||||
|
||||
def user_api_key_auth_from_request_data(request_data: Mapping[str, object]) -> "UserAPIKeyAuth":
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
|
||||
stamped: Final[Mapping[str, object]] = metadata if isinstance(metadata, dict) else {}
|
||||
stamped: Final = _stamped_metadata(request_data)
|
||||
|
||||
def stamped_str(field: str) -> str | None:
|
||||
value: Final = stamped.get(field)
|
||||
|
|
@ -915,7 +938,7 @@ class CustomGuardrail(CustomLogger):
|
|||
if target is not self:
|
||||
kwargs["guardrail_to_apply"] = self
|
||||
result: Final = await target.async_pre_call_hook(
|
||||
user_api_key_dict=_user_api_key_auth_from_request(kwargs),
|
||||
user_api_key_dict=user_api_key_auth_from_request_data(kwargs),
|
||||
cache=dc,
|
||||
data=kwargs,
|
||||
call_type="completion" if call_type == CallTypes.completion else "acompletion",
|
||||
|
|
@ -946,7 +969,7 @@ class CustomGuardrail(CustomLogger):
|
|||
}
|
||||
try:
|
||||
result: Final = await target.async_pre_call_hook(
|
||||
user_api_key_dict=_user_api_key_auth_from_request(scan_request),
|
||||
user_api_key_dict=user_api_key_auth_from_request_data(scan_request),
|
||||
cache=dc,
|
||||
data=scan_request,
|
||||
call_type="acompletion",
|
||||
|
|
@ -996,7 +1019,7 @@ class CustomGuardrail(CustomLogger):
|
|||
if target is not self:
|
||||
request_data["guardrail_to_apply"] = self # rebind-ok: dispatch consumes this key
|
||||
result: Final = await target.async_post_call_success_hook(
|
||||
user_api_key_dict=_user_api_key_auth_from_request(request_data),
|
||||
user_api_key_dict=user_api_key_auth_from_request_data(request_data),
|
||||
data=request_data,
|
||||
response=response,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
|
|
@ -21,6 +22,7 @@ from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
|
|||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
user_api_key_auth_from_request_data,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
get_attribute_or_key,
|
||||
|
|
@ -28,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
group_tool_exchanges,
|
||||
has_tool_with_name,
|
||||
)
|
||||
from litellm.litellm_core_utils.provider_affinity import get_stable_session_id
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -56,6 +59,8 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
BYPASS_HEADER: Final = "x-headroom-bypass"
|
||||
AFFINITY_HEADER: Final = "x-headroom-affinity"
|
||||
_AFFINITY_KEY_LENGTH: Final = 32
|
||||
_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
|
||||
(CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
|
||||
)
|
||||
|
|
@ -87,6 +92,27 @@ def _is_object_list(value: object) -> TypeGuard[list[object]]: # guard-ok: isin
|
|||
return isinstance(value, list)
|
||||
|
||||
|
||||
def _affinity_key(request_data: Mapping[str, object]) -> str | None:
|
||||
"""Opaque routing value for related Headroom calls, or None.
|
||||
|
||||
Both halves must be present: without a session id there is nothing to keep
|
||||
together, and without a key the proxy authenticated the same session id could
|
||||
belong to different tenants. Neither raw value leaves the proxy.
|
||||
"""
|
||||
session_id: Final = get_stable_session_id(request_data)
|
||||
key_hash: Final = user_api_key_auth_from_request_data(request_data).api_key
|
||||
if not session_id or not key_hash:
|
||||
return None
|
||||
return hashlib.sha256(f"{key_hash}|{session_id}".encode()).hexdigest()[:_AFFINITY_KEY_LENGTH]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _IssuedHashes:
|
||||
hashes: frozenset[str]
|
||||
expires_at: float
|
||||
affinity: str | None
|
||||
|
||||
|
||||
def _flatten_messages_for_compression(messages: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
"""Collapse all-text list-of-parts content to plain strings for /v1/compress.
|
||||
|
||||
|
|
@ -521,7 +547,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
self.async_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback,
|
||||
)
|
||||
self._issued_hashes_by_call_id: dict[str, tuple[frozenset[str], float]] = {}
|
||||
self._issued_hashes_by_call_id: dict[str, _IssuedHashes] = {}
|
||||
super().__init__( # pyright: ignore[reportUnknownMemberType]
|
||||
guardrail_name=guardrail_name,
|
||||
event_hook=event_hook,
|
||||
|
|
@ -540,10 +566,12 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
value: Final = headers.get(BYPASS_HEADER)
|
||||
return str(value).lower() == "true"
|
||||
|
||||
def _request_headers(self) -> dict[str, str]:
|
||||
def _request_headers(self, affinity: str | None = None) -> dict[str, str]:
|
||||
headers: Final[dict[str, str]] = {"Content-Type": "application/json"}
|
||||
if self.headroom_api_key:
|
||||
headers["Authorization"] = f"Bearer {self.headroom_api_key}"
|
||||
if affinity:
|
||||
headers[AFFINITY_HEADER] = affinity
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -572,9 +600,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
def _prune_expired_hashes(self) -> None:
|
||||
now: Final = time.monotonic()
|
||||
self._issued_hashes_by_call_id = {
|
||||
call_id: (hashes, expiry)
|
||||
for call_id, (hashes, expiry) in self._issued_hashes_by_call_id.items()
|
||||
if expiry > now
|
||||
call_id: issued for call_id, issued in self._issued_hashes_by_call_id.items() if issued.expires_at > now
|
||||
}
|
||||
|
||||
def _handle_compress_failure(
|
||||
|
|
@ -596,6 +622,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
self,
|
||||
messages: list[dict[str, object]],
|
||||
model: str | None,
|
||||
affinity: str | None = None,
|
||||
) -> _CompressResult:
|
||||
payload: Final[dict[str, object]] = {"messages": messages}
|
||||
if model:
|
||||
|
|
@ -605,7 +632,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
raw_response: HttpxResponse = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.post is untyped
|
||||
url=f"{self.headroom_api_base}/v1/compress",
|
||||
json=payload,
|
||||
headers=self._request_headers(),
|
||||
headers=self._request_headers(affinity),
|
||||
timeout=self.timeout,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
|
|
@ -734,7 +761,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
stats["tokens_saved"] = tokens_before - tokens_after
|
||||
return _CompressResult(filtered, True, stats, _read_ccr_hashes(body))
|
||||
|
||||
async def _call_retrieve(self, hash_value: str, query: str | None = None) -> str:
|
||||
async def _call_retrieve(self, hash_value: str, query: str | None = None, affinity: str | None = None) -> str:
|
||||
params: Final[dict[str, str]] = {}
|
||||
if query:
|
||||
params["query"] = query
|
||||
|
|
@ -743,7 +770,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
raw_response: HttpxResponse = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.get is untyped
|
||||
url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}",
|
||||
params=params,
|
||||
headers=self._request_headers(),
|
||||
headers=self._request_headers(affinity),
|
||||
timeout=self.timeout,
|
||||
)
|
||||
except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError, litellm.Timeout) as e:
|
||||
|
|
@ -811,18 +838,20 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
# user message, so it is withheld here at some cost to history ranking.
|
||||
# request_data is a bare dict on the base signature; narrow it before
|
||||
# reading the untranslated messages so long tool names can be recovered.
|
||||
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
|
||||
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
|
||||
narrowed_request_data: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data)
|
||||
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(narrowed_request_data.get("messages"))
|
||||
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
|
||||
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
|
||||
if not compressible:
|
||||
return inputs
|
||||
|
||||
model: Final = self.headroom_model or request_data.get("model")
|
||||
affinity: Final = _affinity_key(narrowed_request_data)
|
||||
start_time: Final = time.time()
|
||||
result: Final = await self._call_compress(
|
||||
messages=_flatten_messages_for_compression(compressible),
|
||||
model=model if isinstance(model, str) else None,
|
||||
affinity=affinity,
|
||||
)
|
||||
end_time: Final = time.time()
|
||||
|
||||
|
|
@ -873,7 +902,11 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
if not call_id:
|
||||
call_id = str(uuid.uuid4())
|
||||
request_data["litellm_call_id"] = call_id
|
||||
self._issued_hashes_by_call_id[call_id] = (frozenset(hashes), time.monotonic() + _HASH_CACHE_TTL_SECONDS)
|
||||
self._issued_hashes_by_call_id[call_id] = _IssuedHashes(
|
||||
hashes=frozenset(hashes),
|
||||
expires_at=time.monotonic() + _HASH_CACHE_TTL_SECONDS,
|
||||
affinity=affinity,
|
||||
)
|
||||
|
||||
existing_tools: Final = inputs.get("tools")
|
||||
retrieve_tool: Final = _build_headroom_retrieve_tool()
|
||||
|
|
@ -936,7 +969,9 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
|
||||
self._prune_expired_hashes()
|
||||
call_id: Final = _resolve_call_id(logging_obj, kwargs)
|
||||
valid_hashes = self._issued_hashes_by_call_id.get(call_id, (frozenset(), 0.0))[0] if call_id else frozenset()
|
||||
issued: Final = self._issued_hashes_by_call_id.get(call_id) if call_id else None
|
||||
valid_hashes: Final = issued.hashes if issued else frozenset()
|
||||
affinity: Final = issued.affinity if issued else None
|
||||
|
||||
retrieved: Final[list[tuple[dict[str, object], str]]] = []
|
||||
for tc in tool_calls:
|
||||
|
|
@ -959,6 +994,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
content = await self._call_retrieve(
|
||||
hash_value=hash_value,
|
||||
query=str(query) if query else None,
|
||||
affinity=affinity,
|
||||
)
|
||||
verbose_proxy_logger.debug("Headroom CCR: retrieved hash=%s (%d chars)", hash_value, len(content))
|
||||
retrieved.append((tc, content))
|
||||
|
|
|
|||
129
tests/integration/observability/test_headroom_affinity.py
Normal file
129
tests/integration/observability/test_headroom_affinity.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
"""The Headroom guardrail tags its outbound calls with an opaque session affinity, through a real proxy.
|
||||
|
||||
`x-headroom-affinity` is derived from the validated virtual key and the client's own session id and
|
||||
nothing else, so the compression service can pin related calls together without ever seeing either
|
||||
raw value. The value must repeat for the same key and session, change when the key changes, ignore
|
||||
identity fields a client forges into the request body, and be absent when the client sent no session
|
||||
id. Everything is asserted on the requests the owned compression sink recorded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, object_value
|
||||
from integration._support.openai_wire import chat_reply
|
||||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
AFFINITY_HEADER: Final = "x-headroom-affinity"
|
||||
SESSION_HEADER: Final = "x-litellm-session-id"
|
||||
COMPRESS_PATH: Final = "/v1/compress"
|
||||
AFFINITY_SHAPE: Final = re.compile(r"[0-9a-f]{32}")
|
||||
SESSION_ID: Final = "3f1c9b7e2d4a4c6e8b0f1a2c3d4e5f60"
|
||||
FORGED_IDENTITY: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"user_api_key_team_id": "victim", "user_api_key_hash": "forged"}
|
||||
)
|
||||
MESSAGES: Final[tuple[dict[str, JsonValue], ...]] = (
|
||||
{"role": "system", "content": "You are a terse assistant."},
|
||||
{"role": "user", "content": "Summarize what session affinity buys a compression service."},
|
||||
{"role": "assistant", "content": "It keeps related calls on one node so cached context is reused."},
|
||||
{"role": "user", "content": "And what identifies the session to that service?"},
|
||||
)
|
||||
_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
||||
_JSON: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _compress(request: Request) -> Reply:
|
||||
body: Final = _JSON.validate_json(request.body)
|
||||
return Reply(body=json.dumps({"messages": body["messages"], "tokens_before": 1000, "tokens_after": 100}).encode())
|
||||
|
||||
|
||||
def _provider(_request: Request) -> Reply:
|
||||
return chat_reply("chatcmpl-headroom-affinity", "gpt-4o-mini", "ok", stream=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
owned: OwnedProxy
|
||||
compress: Wire
|
||||
chat_model: str
|
||||
key_a: str
|
||||
key_b: str
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("headroom-affinity")
|
||||
with wire_server(_compress) as compress, wire_server(_provider) as provider:
|
||||
config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text()))
|
||||
path: Final = directory / "headroom-affinity.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**config,
|
||||
"litellm_settings": {**object_value(config["litellm_settings"]), "cache": False},
|
||||
"guardrails": [
|
||||
{
|
||||
"guardrail_name": "headroom",
|
||||
"litellm_params": {
|
||||
"guardrail": "headroom",
|
||||
"mode": "pre_call",
|
||||
"default_on": True,
|
||||
"api_base": compress.url,
|
||||
"api_key": "synthetic-headroom-key",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
with (
|
||||
owned_proxy_process(session_gateway, directory, {}, config=path) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
chat_model: Final = scenario.model(api_base=provider.url + "/v1")
|
||||
yield Rig(owned, compress, chat_model, scenario.key(), scenario.key())
|
||||
|
||||
|
||||
def _affinity_sent(
|
||||
rig: Rig, key: str, *, session: str | None, body: Mapping[str, JsonValue] = MappingProxyType({})
|
||||
) -> str | None:
|
||||
rig.compress.drain()
|
||||
response: Final = rig.owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": list(MESSAGES), **body},
|
||||
key=key,
|
||||
headers={SESSION_HEADER: session} if session is not None else None,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
compress_calls: Final = tuple(request for request in rig.compress.drain() if request.target == COMPRESS_PATH)
|
||||
assert len(compress_calls) == 1, [request.target for request in compress_calls]
|
||||
return compress_calls[0].headers.get(AFFINITY_HEADER)
|
||||
|
||||
|
||||
def test_affinity_repeats_per_key_and_session_without_exposing_either(rig: Rig) -> None:
|
||||
first: Final = _affinity_sent(rig, rig.key_a, session=SESSION_ID)
|
||||
assert first is not None and AFFINITY_SHAPE.fullmatch(first), first
|
||||
assert _affinity_sent(rig, rig.key_a, session=SESSION_ID) == first
|
||||
assert first not in SESSION_ID
|
||||
assert first not in sha256(rig.key_a.encode()).hexdigest()
|
||||
assert _affinity_sent(rig, rig.key_a, session=SESSION_ID, body=FORGED_IDENTITY) == first
|
||||
other_key: Final = _affinity_sent(rig, rig.key_b, session=SESSION_ID)
|
||||
assert other_key is not None and AFFINITY_SHAPE.fullmatch(other_key), other_key
|
||||
assert other_key != first
|
||||
|
||||
|
||||
def test_no_client_session_id_sends_no_affinity(rig: Rig) -> None:
|
||||
assert _affinity_sent(rig, rig.key_a, session=None) is None
|
||||
|
|
@ -15,6 +15,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
_request_is_streaming,
|
||||
guardrail_request_data_with_streaming,
|
||||
log_guardrail_information,
|
||||
user_api_key_auth_from_request_data,
|
||||
without_server_streaming_classification,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
|
@ -3786,3 +3787,41 @@ def test_default_on_guardrail() -> None:
|
|||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
_STAMPED_IDENTITY: Final = {
|
||||
"user_api_key_hash": "hash-a",
|
||||
"user_api_key_team_id": "team-a",
|
||||
"user_api_key_user_id": "user-a",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data",
|
||||
[
|
||||
pytest.param({"model": "m", "metadata": _STAMPED_IDENTITY}, id="chat route"),
|
||||
pytest.param(
|
||||
{"model": "m", "metadata": _STAMPED_IDENTITY, "litellm_metadata": {"user_api_key_token": "hash-a"}},
|
||||
id="chat route after the unified guardrail adds litellm_metadata",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "m", "litellm_metadata": _STAMPED_IDENTITY, "metadata": {"user_api_key_hash": "forged"}},
|
||||
id="messages route with caller metadata",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_user_api_key_auth_from_request_data_reads_the_stamped_container(request_data: dict[str, object]) -> None:
|
||||
auth: Final = user_api_key_auth_from_request_data(request_data)
|
||||
|
||||
assert (auth.api_key, auth.team_id, auth.user_id) == ("hash-a", "team-a", "user-a")
|
||||
|
||||
|
||||
def test_user_api_key_auth_from_request_data_without_a_stamped_hash_keeps_the_route_container() -> None:
|
||||
auth: Final = user_api_key_auth_from_request_data(
|
||||
{
|
||||
"litellm_metadata": {"user_api_key_user_id": "route-user"},
|
||||
"metadata": {"user_api_key_user_id": "other-user"},
|
||||
}
|
||||
)
|
||||
|
||||
assert (auth.api_key, auth.user_id) == (None, "route-user")
|
||||
|
|
|
|||
|
|
@ -19,10 +19,17 @@ Tests cover:
|
|||
- CCR: async_build_agentic_loop_plan calls retrieve endpoint and builds follow-up messages
|
||||
- CCR: streaming /chat/completions is converted to a non-streaming call so the agentic
|
||||
loop resolves the retrieve tool call, then fake-streamed back to the client
|
||||
- session affinity: an explicit session id plus the key hash the proxy stamped
|
||||
produce a stable x-headroom-affinity header on /v1/compress, /v1/retrieve replays
|
||||
the value stored with the issued hashes, and the header is omitted whenever
|
||||
either half is missing
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -32,11 +39,12 @@ import respx
|
|||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
|
||||
HeadroomGuardrail,
|
||||
has_headroom_retrieve_tool,
|
||||
AFFINITY_HEADER,
|
||||
HEADROOM_RETRIEVE_TOOL_NAME,
|
||||
HeadroomGuardrail,
|
||||
_IssuedHashes,
|
||||
has_headroom_retrieve_tool,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.compression_savings import (
|
||||
extract_compression_saved_tokens,
|
||||
|
|
@ -508,9 +516,10 @@ async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages(
|
|||
tool_id="call_abc123",
|
||||
)
|
||||
messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = (
|
||||
frozenset({"b573993006976af767214fac"}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
|
||||
hashes=frozenset({"b573993006976af767214fac"}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
|
|
@ -574,9 +583,10 @@ async def test_async_build_agentic_loop_plan_handles_retrieve_404(
|
|||
"content": "Retrieve more: hash=deadbeef000000000000dead",
|
||||
}
|
||||
]
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = (
|
||||
frozenset({"deadbeef000000000000dead"}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
|
||||
hashes=frozenset({"deadbeef000000000000dead"}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
|
|
@ -658,9 +668,10 @@ async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_c
|
|||
request just because the second request echoes that hash-shaped string
|
||||
back in its own messages -- retrieval must be scoped per litellm_call_id,
|
||||
not derived by re-scanning attacker-controlled message text."""
|
||||
guardrail._issued_hashes_by_call_id["call-A"] = (
|
||||
frozenset({"b573993006976af767214fac"}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["call-A"] = _IssuedHashes(
|
||||
hashes=frozenset({"b573993006976af767214fac"}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
tool_calls = [
|
||||
|
|
@ -734,9 +745,10 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_
|
|||
}
|
||||
]
|
||||
messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = (
|
||||
frozenset({"b573993006976af767214fac"}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
|
||||
hashes=frozenset({"b573993006976af767214fac"}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
|
|
@ -808,9 +820,10 @@ async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messag
|
|||
}
|
||||
]
|
||||
messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = (
|
||||
frozenset({"b573993006976af767214fac"}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
|
||||
hashes=frozenset({"b573993006976af767214fac"}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
|
|
@ -900,8 +913,8 @@ async def test_service_declared_ccr_hashes_drive_injection_and_validation(guardr
|
|||
)
|
||||
|
||||
assert has_headroom_retrieve_tool(result.get("tools") or [])
|
||||
((issued, _expiry),) = guardrail._issued_hashes_by_call_id.values()
|
||||
assert issued == frozenset({"98ca69107318", "b573993006976af767214fac"})
|
||||
(issued,) = guardrail._issued_hashes_by_call_id.values()
|
||||
assert issued.hashes == frozenset({"98ca69107318", "b573993006976af767214fac"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -2313,9 +2326,10 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end(
|
|||
to a client that never declared the tool."""
|
||||
original_content = "the full uncompressed document"
|
||||
final_answer = "the document says hello"
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
|
||||
frozenset({CCR_HASH}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes(
|
||||
hashes=frozenset({CCR_HASH}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
|
|
@ -2367,9 +2381,10 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end(
|
|||
instead of streaming a headroom_retrieve function_call to the client."""
|
||||
original_content = "the full uncompressed document"
|
||||
final_answer = "the document says hello"
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
|
||||
frozenset({CCR_HASH}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes(
|
||||
hashes=frozenset({CCR_HASH}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
|
|
@ -2423,9 +2438,10 @@ def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end(
|
|||
completed response object."""
|
||||
original_content = "the full uncompressed document"
|
||||
final_answer = "the document says hello"
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
|
||||
frozenset({CCR_HASH}),
|
||||
time.monotonic() + 999,
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = _IssuedHashes(
|
||||
hashes=frozenset({CCR_HASH}),
|
||||
expires_at=time.monotonic() + 999,
|
||||
affinity=None,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
|
|
@ -2849,7 +2865,9 @@ def _anthropic_response_with_text_and_tool_call() -> dict:
|
|||
|
||||
|
||||
async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list):
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({"h" * 24}), time.monotonic() + 60)
|
||||
guardrail._issued_hashes_by_call_id["call-1"] = _IssuedHashes(
|
||||
hashes=frozenset({"h" * 24}), expires_at=time.monotonic() + 60, affinity=None
|
||||
)
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.litellm_call_id = "call-1"
|
||||
logging_obj.model_call_details = {}
|
||||
|
|
@ -3075,3 +3093,170 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float):
|
|||
|
||||
assert guardrail.timeout.read == 60.0
|
||||
assert guardrail.timeout.connect == 5.0
|
||||
|
||||
|
||||
CHAT_REQUEST: Final[Mapping[str, object]] = {
|
||||
"model": "gpt-4o",
|
||||
"litellm_session_id": "sess-abc",
|
||||
"metadata": {"user_api_key_hash": "hash-a"},
|
||||
}
|
||||
MESSAGES_REQUEST: Final[Mapping[str, object]] = {
|
||||
"model": "gpt-4o",
|
||||
"litellm_session_id": "sess-abc",
|
||||
"litellm_metadata": {"user_api_key_hash": "hash-a"},
|
||||
}
|
||||
FORGED_IDENTITY: Final[Mapping[str, object]] = {"user_api_key_hash": "forged", "user_api_key_team_id": "victim"}
|
||||
|
||||
|
||||
def _affinity_of(call: respx.models.Call) -> str | None:
|
||||
value: Final = call.request.headers.get(AFFINITY_HEADER)
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def _captured_affinity(route: respx.Route) -> str | None:
|
||||
return _affinity_of(route.calls[-1])
|
||||
|
||||
|
||||
def _mock_compress(respx_mock: respx.MockRouter, ccr_hashes: Sequence[str] = ()) -> respx.Route:
|
||||
messages: Final = COMPRESSED_MESSAGES_WITH_HASH if ccr_hashes else COMPRESSED_MESSAGES
|
||||
return respx_mock.post(f"{FAKE_API_BASE}/v1/compress").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"messages": messages, "tokens_before": 1000, "tokens_after": 100, "ccr_hashes": list(ccr_hashes)},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _compress_with(guardrail: HeadroomGuardrail, request_data: Mapping[str, object]) -> None:
|
||||
await guardrail.apply_guardrail(
|
||||
inputs=GenericGuardrailAPIInputs(texts=["A" * 5000], structured_messages=ORIGINAL_MESSAGES),
|
||||
request_data=copy.deepcopy(dict(request_data)),
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compress_sends_stable_affinity_for_one_conversation(
|
||||
httpx_transport: None,
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = _mock_compress(respx_mock)
|
||||
|
||||
await _compress_with(guardrail, CHAT_REQUEST)
|
||||
await _compress_with(guardrail, CHAT_REQUEST)
|
||||
|
||||
first, second = tuple(_affinity_of(call) for call in route.calls)
|
||||
assert first is not None
|
||||
assert second == first
|
||||
assert re.fullmatch(r"[0-9a-f]{32}", first)
|
||||
assert "sess-abc" not in first
|
||||
assert "hash-a" not in first
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_affinity_differs_per_key_for_the_same_session_id(
|
||||
httpx_transport: None,
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = _mock_compress(respx_mock)
|
||||
|
||||
await _compress_with(guardrail, CHAT_REQUEST)
|
||||
await _compress_with(guardrail, {**CHAT_REQUEST, "metadata": {"user_api_key_hash": "hash-b"}})
|
||||
|
||||
key_a, key_b = tuple(_affinity_of(call) for call in route.calls)
|
||||
assert key_a is not None
|
||||
assert key_b is not None
|
||||
assert key_b != key_a
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_data",
|
||||
[
|
||||
{"model": "gpt-4o", "metadata": {"user_api_key_hash": "hash-a"}},
|
||||
{**CHAT_REQUEST, "metadata": {"litellm_session_id_generated": True, "user_api_key_hash": "hash-a"}},
|
||||
{**MESSAGES_REQUEST, "litellm_metadata": {"litellm_session_id_generated": True, "user_api_key_hash": "hash-a"}},
|
||||
{"model": "gpt-4o", "litellm_session_id": "sess-abc"},
|
||||
],
|
||||
ids=["no_session_id", "generated_session_id_chat", "generated_session_id_messages", "no_stamped_identity"],
|
||||
)
|
||||
async def test_affinity_omitted_when_either_half_is_missing(
|
||||
httpx_transport: None,
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
request_data: Mapping[str, object],
|
||||
) -> None:
|
||||
route: Final = _mock_compress(respx_mock)
|
||||
|
||||
await _compress_with(guardrail, request_data)
|
||||
|
||||
assert _captured_affinity(route) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_affinity_ignores_identity_a_client_puts_in_the_body(
|
||||
httpx_transport: None,
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = _mock_compress(respx_mock)
|
||||
|
||||
await _compress_with(guardrail, CHAT_REQUEST)
|
||||
await _compress_with(guardrail, {**CHAT_REQUEST, **FORGED_IDENTITY})
|
||||
await _compress_with(guardrail, {"model": "gpt-4o", "litellm_session_id": "sess-abc", **FORGED_IDENTITY})
|
||||
|
||||
stamped, forged_beside_stamped, forged_alone = tuple(_affinity_of(call) for call in route.calls)
|
||||
assert stamped is not None
|
||||
assert forged_beside_stamped == stamped
|
||||
assert forged_alone is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_affinity_reads_identity_from_the_container_the_route_stamps(
|
||||
httpx_transport: None,
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = _mock_compress(respx_mock)
|
||||
|
||||
await _compress_with(guardrail, CHAT_REQUEST)
|
||||
await _compress_with(guardrail, MESSAGES_REQUEST)
|
||||
await _compress_with(guardrail, {**CHAT_REQUEST, "litellm_metadata": {"user_api_key_token": "hash-a"}})
|
||||
|
||||
chat, messages, chat_through_unified_guardrail = tuple(_affinity_of(call) for call in route.calls)
|
||||
assert chat is not None
|
||||
assert messages == chat
|
||||
assert chat_through_unified_guardrail == chat
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve_replays_the_affinity_the_compress_call_sent(
|
||||
httpx_transport: None,
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
compress: Final = _mock_compress(respx_mock, ccr_hashes=(CCR_HASH,))
|
||||
retrieve: Final = respx_mock.get(f"{FAKE_API_BASE}/v1/retrieve/{CCR_HASH}").mock(
|
||||
return_value=httpx.Response(200, json={"original_content": "restored"})
|
||||
)
|
||||
|
||||
await _compress_with(guardrail, {**CHAT_REQUEST, "litellm_call_id": "call-1"})
|
||||
compress_affinity: Final = _captured_affinity(compress)
|
||||
assert compress_affinity is not None
|
||||
|
||||
await guardrail.async_build_agentic_loop_plan(
|
||||
tools={"tool_calls": [{"arguments": {"hash": CCR_HASH}}]},
|
||||
model="gpt-4o",
|
||||
messages=list(ORIGINAL_MESSAGES),
|
||||
response={},
|
||||
anthropic_messages_provider_config=None,
|
||||
anthropic_messages_optional_request_params={},
|
||||
logging_obj=None,
|
||||
stream=False,
|
||||
kwargs={"litellm_call_id": "call-1"},
|
||||
)
|
||||
|
||||
assert retrieve.called
|
||||
assert _captured_affinity(retrieve) == compress_affinity
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue