mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 566f2ed6fc into 96792d0913
This commit is contained in:
commit
89b82fc04d
9 changed files with 323 additions and 12 deletions
|
|
@ -19,6 +19,7 @@ from enum import Enum
|
|||
from typing import Any, Final, Literal, cast
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import current_service_target, service_target
|
||||
|
|
@ -73,6 +74,12 @@ class CacheMode(str, Enum):
|
|||
|
||||
|
||||
#### LiteLLM.Completion / Embedding Cache ####
|
||||
def _is_str_keyed_mapping(
|
||||
value: object,
|
||||
) -> TypeIs[Mapping[str, object]]: # guard-ok: request kwargs and their metadata dicts are str-keyed
|
||||
return isinstance(value, Mapping)
|
||||
|
||||
|
||||
def _is_conversation_item(item: object) -> bool:
|
||||
if isinstance(item, BaseModel):
|
||||
return True
|
||||
|
|
@ -142,6 +149,7 @@ class Cache:
|
|||
semantic_cache_embedding_max_input_tokens: int | None = None,
|
||||
semantic_cache_embedding_timeout: float | None = None,
|
||||
semantic_cache_scope: str = SemanticCacheScope.KEY.value,
|
||||
cache_scope: str = CacheScope.GLOBAL.value,
|
||||
max_messages: int | None = 4,
|
||||
# GCP IAM authentication parameters
|
||||
gcp_service_account: str | None = None,
|
||||
|
|
@ -172,6 +180,7 @@ class Cache:
|
|||
semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
|
||||
semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
|
||||
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
|
||||
cache_scope (str, optional): Who can share a cache hit. "global" shares hits between every caller. "team" isolates hits per team, and keys without a team per key. "key" isolates hits per key. On the proxy, a non-global scope also ignores caller-supplied `cache_key` / `preset_cache_key`. Defaults to "global".
|
||||
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
|
||||
|
||||
# Disk Cache Args
|
||||
|
|
@ -323,6 +332,7 @@ class Cache:
|
|||
self.ttl = ttl
|
||||
self.mode: CacheMode = mode or CacheMode.default_on
|
||||
self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
|
||||
self.cache_scope: str = CacheScope(cache_scope).value
|
||||
self.max_messages: int | None = max_messages
|
||||
|
||||
if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
|
||||
|
|
@ -365,17 +375,34 @@ class Cache:
|
|||
return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
|
||||
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
|
||||
|
||||
def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
|
||||
litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
|
||||
metadata_sources: Final[tuple[dict, ...]] = tuple(
|
||||
source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
|
||||
@staticmethod
|
||||
def _auth_metadata_value(kwargs: Mapping[str, object], field: str) -> object | None:
|
||||
litellm_params: Final = kwargs.get("litellm_params")
|
||||
containers: Final[tuple[Mapping[str, object], ...]] = (
|
||||
(kwargs, litellm_params) if _is_str_keyed_mapping(litellm_params) else (kwargs,)
|
||||
)
|
||||
candidates: Final = (container.get(key) for container in containers for key in ("metadata", "litellm_metadata"))
|
||||
metadata_sources: Final = tuple(source for source in candidates if _is_str_keyed_mapping(source))
|
||||
return next((source[field] for source in metadata_sources if source.get(field) is not None), None)
|
||||
|
||||
def _get_semantic_cache_tenant_scope(self, kwargs: Mapping[str, object]) -> str:
|
||||
scope_values: Final = (
|
||||
(field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
|
||||
for field in self._semantic_cache_scope_fields()
|
||||
(field, self._auth_metadata_value(kwargs, field)) for field in self._semantic_cache_scope_fields()
|
||||
)
|
||||
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
|
||||
|
||||
def _get_cache_scope_material(self, kwargs: Mapping[str, object]) -> str:
|
||||
if self.cache_scope == CacheScope.GLOBAL:
|
||||
return ""
|
||||
team_id: Final = self._auth_metadata_value(kwargs, "user_api_key_team_id")
|
||||
if self.cache_scope == CacheScope.TEAM and team_id is not None:
|
||||
return f"cache_scope_team: {team_id}"
|
||||
key_hash: Final = self._auth_metadata_value(kwargs, "user_api_key_hash")
|
||||
return "" if key_hash is None else f"cache_scope_key: {key_hash}"
|
||||
|
||||
def lacks_cache_scope_identity(self, kwargs: Mapping[str, object]) -> bool:
|
||||
return self.cache_scope != CacheScope.GLOBAL and self._get_cache_scope_material(kwargs) == ""
|
||||
|
||||
def get_cache_key(self, **kwargs: object) -> str:
|
||||
"""
|
||||
Get the cache key for the given arguments.
|
||||
|
|
@ -419,7 +446,8 @@ class Cache:
|
|||
reasoning_summary_material: Final = (
|
||||
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
|
||||
cache_scope_material: Final = self._get_cache_scope_material(kwargs)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + cache_scope_material + reasoning_summary_material)
|
||||
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
|
||||
verbose_logger.debug(
|
||||
"\nCreated cache key: %s (source material length: %d)",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import litellm
|
|||
from litellm._internal_context import is_internal_call, post_response_phase
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
from litellm.caching import InMemoryCache
|
||||
from litellm.caching.caching import S3Cache, response_cache_phase
|
||||
from litellm.caching.caching import Cache, S3Cache, response_cache_phase
|
||||
from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS, RESPONSE_CACHE_EXCLUDED_PROVIDERS
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
||||
|
|
@ -115,6 +115,7 @@ def _is_response_cache_excluded(model: str | None, kwargs: Mapping[str, object])
|
|||
(isinstance(custom_llm_provider, str) and custom_llm_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS)
|
||||
or model_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS
|
||||
or is_github_copilot_per_user_request(kwargs)
|
||||
or (isinstance(litellm.cache, Cache) and litellm.cache.lacks_cache_scope_identity(kwargs))
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from fastapi import HTTPException, Request, status
|
|||
from pydantic import TypeAdapter
|
||||
from pydantic import ValidationError as PydanticValidationError
|
||||
from starlette.datastructures import Headers
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger, verbose_proxy_logger
|
||||
|
|
@ -87,6 +88,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
safe_get_request_headers,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata
|
||||
from litellm.types.caching import CacheScope
|
||||
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
|
||||
|
||||
# Cache special headers as a frozenset for O(1) lookup performance
|
||||
|
|
@ -808,6 +810,33 @@ def _strip_router_reserved_metadata(
|
|||
)
|
||||
|
||||
|
||||
_CLIENT_CACHE_KEY_OVERRIDE_FIELDS: Final = frozenset({"cache_key", "preset_cache_key"})
|
||||
|
||||
|
||||
def _is_request_json_object(
|
||||
value: object,
|
||||
) -> TypeIs[dict[str, object]]: # guard-ok: parsed JSON request bodies have str keys # mutable-ok: stripped in place
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _strip_client_cache_key_overrides(
|
||||
data: dict[str, object], # mutable-ok: strips in place on the request body the pre-call pipeline threads through
|
||||
) -> None:
|
||||
"""Under a scoped response cache, the cache key must come from proxy auth, never from the caller."""
|
||||
if litellm.cache is None or litellm.cache.cache_scope == CacheScope.GLOBAL:
|
||||
return
|
||||
for field in _CLIENT_CACHE_KEY_OVERRIDE_FIELDS & data.keys():
|
||||
data.pop(field)
|
||||
if not _is_request_json_object(litellm_params := data.get("litellm_params")):
|
||||
return
|
||||
litellm_params.pop("preset_cache_key", None)
|
||||
for metadata_key in ("metadata", "litellm_metadata"):
|
||||
if not _is_request_json_object(metadata := litellm_params.get(metadata_key)):
|
||||
continue
|
||||
for field in tuple(key for key in metadata if key.startswith("user_api_key_")):
|
||||
metadata.pop(field)
|
||||
|
||||
|
||||
def get_metadata_variable_name(request: Request) -> str:
|
||||
"""
|
||||
Helper to return what the "metadata" field should be called in the request data
|
||||
|
|
@ -2647,16 +2676,18 @@ async def add_litellm_data_to_request(
|
|||
# metadata (sent via multipart/form-data or extra_body) wouldn't be a
|
||||
# dict yet at the earlier strip point and the isinstance(dict) guard
|
||||
# would silently skip the field.
|
||||
request_body: Final = cast("dict[str, object]", data) # cast-ok: data is the untyped request body dict
|
||||
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
|
||||
_strip_client_pricing_overrides(data)
|
||||
_strip_router_reserved_metadata(data)
|
||||
_strip_client_pricing_overrides(request_body)
|
||||
_strip_router_reserved_metadata(request_body)
|
||||
_strip_client_cache_key_overrides(request_body)
|
||||
|
||||
# Same reason as the strips above: runs after the metadata string-to-dict parse
|
||||
# so JSON-string metadata cannot smuggle callback credentials past the dict guard.
|
||||
_strip_client_callback_credentials(data)
|
||||
_strip_client_callback_credentials(request_body)
|
||||
|
||||
if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True:
|
||||
_strip_client_message_redaction_opt_out(data)
|
||||
_strip_client_message_redaction_opt_out(request_body)
|
||||
|
||||
# Fill in the proxy_server_request body snapshot now that metadata has
|
||||
# been parsed. Consumers (standard_logging_payload, lago,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,12 @@ class SemanticCacheScope(str, Enum):
|
|||
END_USER = "end_user"
|
||||
|
||||
|
||||
class CacheScope(str, Enum):
|
||||
GLOBAL = "global"
|
||||
TEAM = "team"
|
||||
KEY = "key"
|
||||
|
||||
|
||||
CachingSupportedCallTypes = Literal[
|
||||
"completion",
|
||||
"acompletion",
|
||||
|
|
|
|||
|
|
@ -200,6 +200,19 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [
|
|||
ui_field_name="Semantic Cache Scope",
|
||||
redis_type="semantic",
|
||||
),
|
||||
CacheSettingsField(
|
||||
field_name="cache_scope",
|
||||
field_type="String",
|
||||
field_value=None,
|
||||
field_description=(
|
||||
"Who can share a cache hit. 'global' shares hits between every caller. 'team' isolates hits per team,"
|
||||
" and keys without a team per key. 'key' isolates hits per key."
|
||||
),
|
||||
field_default="global",
|
||||
options=["global", "team", "key"],
|
||||
ui_field_name="Cache Scope",
|
||||
redis_type=None,
|
||||
),
|
||||
# GCP IAM authentication fields
|
||||
CacheSettingsField(
|
||||
field_name="gcp_service_account",
|
||||
|
|
|
|||
|
|
@ -274,6 +274,148 @@ def test_exact_cache_key_still_includes_prompt():
|
|||
assert key_a != key_b
|
||||
|
||||
|
||||
_TEAM_A_KEY_1: Final = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-a"}
|
||||
_TEAM_A_KEY_2: Final = {"user_api_key_hash": "hash-2", "user_api_key_team_id": "team-a"}
|
||||
_TEAM_B_KEY_3: Final = {"user_api_key_hash": "hash-3", "user_api_key_team_id": "team-b"}
|
||||
_NO_TEAM_KEY_4: Final = {"user_api_key_hash": "hash-4"}
|
||||
_NO_TEAM_KEY_5: Final = {"user_api_key_hash": "hash-5"}
|
||||
|
||||
|
||||
def _scoped_keys(cache_scope: str | None, *identities: dict[str, str]) -> list[str]:
|
||||
cache: Final = Cache(type=LiteLLMCacheType.LOCAL, **({} if cache_scope is None else {"cache_scope": cache_scope}))
|
||||
messages: Final = [{"role": "user", "content": "What color is the sky?"}]
|
||||
return [cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata=dict(i)) for i in identities]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cache_scope", "same_team_shares", "other_team_shares", "teamless_keys_share"),
|
||||
[
|
||||
(None, True, True, True),
|
||||
("global", True, True, True),
|
||||
("team", True, False, False),
|
||||
("key", False, False, False),
|
||||
],
|
||||
)
|
||||
def test_cache_scope_controls_who_shares_an_exact_cache_hit(
|
||||
cache_scope, same_team_shares, other_team_shares, teamless_keys_share
|
||||
):
|
||||
key_1, key_2, key_3, key_4, key_5 = _scoped_keys(
|
||||
cache_scope, _TEAM_A_KEY_1, _TEAM_A_KEY_2, _TEAM_B_KEY_3, _NO_TEAM_KEY_4, _NO_TEAM_KEY_5
|
||||
)
|
||||
assert (key_1 == key_2) is same_team_shares
|
||||
assert (key_1 == key_3) is other_team_shares
|
||||
assert (key_4 == key_5) is teamless_keys_share
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_scope", ["team", "key"])
|
||||
def test_cache_scope_is_stable_for_the_same_caller(cache_scope):
|
||||
first, again = _scoped_keys(cache_scope, _TEAM_A_KEY_1, _TEAM_A_KEY_1)
|
||||
assert first == again
|
||||
|
||||
|
||||
def test_cache_scope_reads_identity_from_litellm_metadata_and_litellm_params():
|
||||
cache: Final = Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team")
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
from_metadata: Final = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata=dict(_TEAM_A_KEY_1))
|
||||
from_litellm_metadata: Final = cache.get_cache_key(
|
||||
model="gpt-4o-mini", messages=messages, litellm_metadata=dict(_TEAM_A_KEY_1)
|
||||
)
|
||||
from_litellm_params: Final = cache.get_cache_key(
|
||||
model="gpt-4o-mini", messages=messages, litellm_params={"metadata": dict(_TEAM_A_KEY_1)}
|
||||
)
|
||||
other_team: Final = cache.get_cache_key(
|
||||
model="gpt-4o-mini", messages=messages, litellm_metadata=dict(_TEAM_B_KEY_3)
|
||||
)
|
||||
assert from_metadata == from_litellm_metadata == from_litellm_params
|
||||
assert from_metadata != other_team
|
||||
|
||||
|
||||
def test_cache_scope_rejects_unknown_value():
|
||||
with pytest.raises(ValueError, match="'org' is not a valid CacheScope"):
|
||||
Cache(type=LiteLLMCacheType.LOCAL, cache_scope="org")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_cache_scope_isolates_acompletion_hits_between_teams(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team"))
|
||||
messages: Final = [{"role": "user", "content": f"team scope {uuid.uuid4().hex}"}]
|
||||
|
||||
async def call(identity: dict[str, str], mock_response: str):
|
||||
return await acompletion(
|
||||
model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response=mock_response
|
||||
)
|
||||
|
||||
team_a_first: Final = await call(_TEAM_A_KEY_1, "a1")
|
||||
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
||||
team_a_other_key: Final = await call(_TEAM_A_KEY_2, "a2")
|
||||
team_b: Final = await call(_TEAM_B_KEY_3, "b")
|
||||
|
||||
assert team_a_other_key.id == team_a_first.id
|
||||
assert team_b.id != team_a_first.id
|
||||
assert team_b.choices[0].message.content == "b"
|
||||
|
||||
|
||||
_NO_IDENTITY: Final = {}
|
||||
_TEAMLESS_JWT_CALLER: Final = {"user_api_key_user_id": "jwt-user"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("cache_scope", "identity"),
|
||||
[("team", _NO_IDENTITY), ("key", _NO_IDENTITY), ("team", _TEAMLESS_JWT_CALLER), ("key", _TEAMLESS_JWT_CALLER)],
|
||||
)
|
||||
async def test_scoped_cache_never_shares_hits_between_callers_without_identity(
|
||||
monkeypatch: pytest.MonkeyPatch, cache_scope: str, identity: dict[str, str]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope))
|
||||
messages: Final = [{"role": "user", "content": f"no identity {uuid.uuid4().hex}"}]
|
||||
|
||||
first: Final = await acompletion(model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response="1")
|
||||
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
||||
second: Final = await acompletion(
|
||||
model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response="2"
|
||||
)
|
||||
|
||||
assert second.id != first.id
|
||||
assert second.choices[0].message.content == "2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scoped_cache_without_identity_does_not_read_a_scoped_callers_entry(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team"))
|
||||
messages: Final = [{"role": "user", "content": f"cross read {uuid.uuid4().hex}"}]
|
||||
|
||||
team_a: Final = await acompletion(
|
||||
model="gpt-4o-mini", messages=messages, metadata=dict(_TEAM_A_KEY_1), mock_response="a"
|
||||
)
|
||||
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
||||
anonymous: Final = await acompletion(model="gpt-4o-mini", messages=messages, mock_response="anon")
|
||||
|
||||
assert anonymous.id != team_a.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_cache_scope_still_serves_cached_embeddings_to_the_same_team(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team"))
|
||||
text: Final = f"embed {uuid.uuid4().hex}"
|
||||
|
||||
await aembedding(model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_A_KEY_1), mock_response=[0.1])
|
||||
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
||||
same_team: Final = await aembedding(
|
||||
model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_A_KEY_2), mock_response=[0.9]
|
||||
)
|
||||
other_team: Final = await aembedding(
|
||||
model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_B_KEY_3), mock_response=[0.5]
|
||||
)
|
||||
|
||||
assert same_team.data[0]["embedding"] == [0.1]
|
||||
assert other_team.data[0]["embedding"] == [0.5]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"anthropic_param",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import copy
|
|||
import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
|
@ -36,6 +37,8 @@ from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_log
|
|||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
|
||||
from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.types.caching import LiteLLMCacheType
|
||||
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
KeyAndTeamLoggingSettings,
|
||||
|
|
@ -1228,6 +1231,69 @@ async def test_add_litellm_data_to_request_keeps_the_request_max_fallbacks_cap()
|
|||
assert updated["max_fallbacks"] == 0
|
||||
|
||||
|
||||
_FORGED_CACHE_KEY_FIELDS: Final = {
|
||||
"cache_key": "ns:victim-hash",
|
||||
"preset_cache_key": "ns:victim-hash",
|
||||
"litellm_params": {
|
||||
"preset_cache_key": "ns:victim-hash",
|
||||
"metadata": {"user_api_key_team_id": "victim-team", "user_api_key_hash": "victim-key"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_FORGED_TOP_LEVEL_CACHE_KEY_ONLY: Final = {"cache_key": "ns:victim-hash", "preset_cache_key": "ns:victim-hash"}
|
||||
|
||||
|
||||
async def _add_data_with_forged_cache_key(
|
||||
monkeypatch: pytest.MonkeyPatch, cache_scope: str, forged: Mapping[str, object] = _FORGED_CACHE_KEY_FIELDS
|
||||
) -> dict:
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope))
|
||||
return await add_litellm_data_to_request(
|
||||
data={
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
**copy.deepcopy(dict(forged)),
|
||||
},
|
||||
request=_make_request_mock("/v1/chat/completions", {"Content-Type": "application/json"}),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", team_id="caller-team"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cache_scope", ["team", "key"])
|
||||
async def test_scoped_cache_ignores_caller_supplied_cache_keys(monkeypatch: pytest.MonkeyPatch, cache_scope: str):
|
||||
updated: Final = await _add_data_with_forged_cache_key(monkeypatch, cache_scope)
|
||||
|
||||
assert "cache_key" not in updated
|
||||
assert "preset_cache_key" not in updated
|
||||
assert "preset_cache_key" not in updated["litellm_params"]
|
||||
assert "user_api_key_team_id" not in updated["litellm_params"]["metadata"]
|
||||
assert "user_api_key_hash" not in updated["litellm_params"]["metadata"]
|
||||
assert updated["metadata"]["user_api_key_team_id"] == "caller-team"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cache_scope", ["team", "key"])
|
||||
async def test_scoped_cache_strips_a_forged_cache_key_on_a_request_without_litellm_params(
|
||||
monkeypatch: pytest.MonkeyPatch, cache_scope: str
|
||||
):
|
||||
updated: Final = await _add_data_with_forged_cache_key(monkeypatch, cache_scope, _FORGED_TOP_LEVEL_CACHE_KEY_ONLY)
|
||||
|
||||
assert "cache_key" not in updated
|
||||
assert "preset_cache_key" not in updated
|
||||
assert "litellm_params" not in updated
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_cache_scope_keeps_caller_supplied_cache_key(monkeypatch: pytest.MonkeyPatch):
|
||||
updated: Final = await _add_data_with_forged_cache_key(monkeypatch, "global")
|
||||
|
||||
assert updated["cache_key"] == "ns:victim-hash"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_litellm_data_to_request_strips_user_control_fields():
|
||||
"""Strip untrusted proxy-control fields before guardrails, logging, and headers read metadata."""
|
||||
|
|
|
|||
|
|
@ -241,6 +241,20 @@ export const CACHE_FIELDS: readonly CacheField[] = [
|
|||
helpText: "Namespace prefix for cache keys",
|
||||
redisType: null,
|
||||
},
|
||||
{
|
||||
name: "cache_scope",
|
||||
label: "Cache Scope",
|
||||
type: "select",
|
||||
section: "cacheManagement",
|
||||
helpText:
|
||||
"Who can share a cache hit. Global shares hits between every caller. Team isolates hits per team, and keys without a team per key. Key isolates hits per key.",
|
||||
redisType: null,
|
||||
options: [
|
||||
{ value: "global", label: "Global (shared by every caller)" },
|
||||
{ value: "team", label: "Team (isolated per team)" },
|
||||
{ value: "key", label: "Key (isolated per key)" },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "ttl",
|
||||
label: "TTL (seconds)",
|
||||
|
|
|
|||
|
|
@ -83,6 +83,16 @@ describe("buildCachePayload", () => {
|
|||
expect(node).not.toHaveProperty("semantic_cache_scope");
|
||||
});
|
||||
|
||||
it("should send the cache scope for every redis type and leave it to the server default when unset", () => {
|
||||
const redisTypes = ["node", "cluster", "sentinel", "semantic"] as const;
|
||||
for (const redisType of redisTypes) {
|
||||
const scoped = buildCachePayload(redisType, { cache_scope: "team" }, { forTesting: false });
|
||||
expect(scoped.cache_scope).toBe("team");
|
||||
}
|
||||
const unset = buildCachePayload("node", buildInitialValues({}), { forTesting: false });
|
||||
expect(unset).not.toHaveProperty("cache_scope");
|
||||
});
|
||||
|
||||
it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => {
|
||||
const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true });
|
||||
expect(payload.type).toBe("redis");
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue