This commit is contained in:
Filipe Andujar 2026-10-26 15:12:06 -03:00 • committed by GitHub
commit 89b82fc04d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 323 additions and 12 deletions

View file

@ -19,6 +19,7 @@ from enum import Enum
from typing import Any, Final, Literal, cast
from pydantic import BaseModel, TypeAdapter
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm._internal_context import current_service_target, service_target
@ -73,6 +74,12 @@ class CacheMode(str, Enum):
#### LiteLLM.Completion / Embedding Cache ####
def _is_str_keyed_mapping(
value: object,
) -> TypeIs[Mapping[str, object]]: # guard-ok: request kwargs and their metadata dicts are str-keyed
return isinstance(value, Mapping)
def _is_conversation_item(item: object) -> bool:
if isinstance(item, BaseModel):
return True
@ -142,6 +149,7 @@ class Cache:
semantic_cache_embedding_max_input_tokens: int | None = None,
semantic_cache_embedding_timeout: float | None = None,
semantic_cache_scope: str = SemanticCacheScope.KEY.value,
cache_scope: str = CacheScope.GLOBAL.value,
max_messages: int | None = 4,
# GCP IAM authentication parameters
gcp_service_account: str | None = None,
@ -172,6 +180,7 @@ class Cache:
semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
cache_scope (str, optional): Who can share a cache hit. "global" shares hits between every caller. "team" isolates hits per team, and keys without a team per key. "key" isolates hits per key. On the proxy, a non-global scope also ignores caller-supplied `cache_key` / `preset_cache_key`. Defaults to "global".
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
# Disk Cache Args
@ -323,6 +332,7 @@ class Cache:
self.ttl = ttl
self.mode: CacheMode = mode or CacheMode.default_on
self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
self.cache_scope: str = CacheScope(cache_scope).value
self.max_messages: int | None = max_messages
if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
@ -365,17 +375,34 @@ class Cache:
return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
metadata_sources: Final[tuple[dict, ...]] = tuple(
source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
@staticmethod
def _auth_metadata_value(kwargs: Mapping[str, object], field: str) -> object | None:
litellm_params: Final = kwargs.get("litellm_params")
containers: Final[tuple[Mapping[str, object], ...]] = (
(kwargs, litellm_params) if _is_str_keyed_mapping(litellm_params) else (kwargs,)
)
candidates: Final = (container.get(key) for container in containers for key in ("metadata", "litellm_metadata"))
metadata_sources: Final = tuple(source for source in candidates if _is_str_keyed_mapping(source))
return next((source[field] for source in metadata_sources if source.get(field) is not None), None)
def _get_semantic_cache_tenant_scope(self, kwargs: Mapping[str, object]) -> str:
scope_values: Final = (
(field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
for field in self._semantic_cache_scope_fields()
(field, self._auth_metadata_value(kwargs, field)) for field in self._semantic_cache_scope_fields()
)
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
def _get_cache_scope_material(self, kwargs: Mapping[str, object]) -> str:
if self.cache_scope == CacheScope.GLOBAL:
return ""
team_id: Final = self._auth_metadata_value(kwargs, "user_api_key_team_id")
if self.cache_scope == CacheScope.TEAM and team_id is not None:
return f"cache_scope_team: {team_id}"
key_hash: Final = self._auth_metadata_value(kwargs, "user_api_key_hash")
return "" if key_hash is None else f"cache_scope_key: {key_hash}"
def lacks_cache_scope_identity(self, kwargs: Mapping[str, object]) -> bool:
return self.cache_scope != CacheScope.GLOBAL and self._get_cache_scope_material(kwargs) == ""
def get_cache_key(self, **kwargs: object) -> str:
"""
Get the cache key for the given arguments.
@ -419,7 +446,8 @@ class Cache:
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
cache_scope_material: Final = self._get_cache_scope_material(kwargs)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + cache_scope_material + reasoning_summary_material)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

View file

@ -27,7 +27,7 @@ import litellm
from litellm._internal_context import is_internal_call, post_response_phase
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache, response_cache_phase
from litellm.caching.caching import Cache, S3Cache, response_cache_phase
from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS, RESPONSE_CACHE_EXCLUDED_PROVIDERS
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
@ -115,6 +115,7 @@ def _is_response_cache_excluded(model: str | None, kwargs: Mapping[str, object])
(isinstance(custom_llm_provider, str) and custom_llm_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS)
or model_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS
or is_github_copilot_per_user_request(kwargs)
or (isinstance(litellm.cache, Cache) and litellm.cache.lacks_cache_scope_identity(kwargs))
)

View file

@ -22,6 +22,7 @@ from fastapi import HTTPException, Request, status
from pydantic import TypeAdapter
from pydantic import ValidationError as PydanticValidationError
from starlette.datastructures import Headers
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
@ -87,6 +88,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
safe_get_request_headers,
)
from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata
from litellm.types.caching import CacheScope
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
# Cache special headers as a frozenset for O(1) lookup performance
@ -808,6 +810,33 @@ def _strip_router_reserved_metadata(
)
_CLIENT_CACHE_KEY_OVERRIDE_FIELDS: Final = frozenset({"cache_key", "preset_cache_key"})
def _is_request_json_object(
value: object,
) -> TypeIs[dict[str, object]]: # guard-ok: parsed JSON request bodies have str keys # mutable-ok: stripped in place
return isinstance(value, dict)
def _strip_client_cache_key_overrides(
data: dict[str, object], # mutable-ok: strips in place on the request body the pre-call pipeline threads through
) -> None:
"""Under a scoped response cache, the cache key must come from proxy auth, never from the caller."""
if litellm.cache is None or litellm.cache.cache_scope == CacheScope.GLOBAL:
return
for field in _CLIENT_CACHE_KEY_OVERRIDE_FIELDS & data.keys():
data.pop(field)
if not _is_request_json_object(litellm_params := data.get("litellm_params")):
return
litellm_params.pop("preset_cache_key", None)
for metadata_key in ("metadata", "litellm_metadata"):
if not _is_request_json_object(metadata := litellm_params.get(metadata_key)):
continue
for field in tuple(key for key in metadata if key.startswith("user_api_key_")):
metadata.pop(field)
def get_metadata_variable_name(request: Request) -> str:
"""
Helper to return what the "metadata" field should be called in the request data
@ -2647,16 +2676,18 @@ async def add_litellm_data_to_request(
# metadata (sent via multipart/form-data or extra_body) wouldn't be a
# dict yet at the earlier strip point and the isinstance(dict) guard
# would silently skip the field.
request_body: Final = cast("dict[str, object]", data) # cast-ok: data is the untyped request body dict
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
_strip_client_pricing_overrides(data)
_strip_router_reserved_metadata(data)
_strip_client_pricing_overrides(request_body)
_strip_router_reserved_metadata(request_body)
_strip_client_cache_key_overrides(request_body)
# Same reason as the strips above: runs after the metadata string-to-dict parse
# so JSON-string metadata cannot smuggle callback credentials past the dict guard.
_strip_client_callback_credentials(data)
_strip_client_callback_credentials(request_body)
if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True:
_strip_client_message_redaction_opt_out(data)
_strip_client_message_redaction_opt_out(request_body)
# Fill in the proxy_server_request body snapshot now that metadata has
# been parsed. Consumers (standard_logging_payload, lago,

View file

@ -24,6 +24,12 @@ class SemanticCacheScope(str, Enum):
END_USER = "end_user"
class CacheScope(str, Enum):
GLOBAL = "global"
TEAM = "team"
KEY = "key"
CachingSupportedCallTypes = Literal[
"completion",
"acompletion",

View file

@ -200,6 +200,19 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [
ui_field_name="Semantic Cache Scope",
redis_type="semantic",
),
CacheSettingsField(
field_name="cache_scope",
field_type="String",
field_value=None,
field_description=(
"Who can share a cache hit. 'global' shares hits between every caller. 'team' isolates hits per team,"
" and keys without a team per key. 'key' isolates hits per key."
),
field_default="global",
options=["global", "team", "key"],
ui_field_name="Cache Scope",
redis_type=None,
),
# GCP IAM authentication fields
CacheSettingsField(
field_name="gcp_service_account",

View file

@ -274,6 +274,148 @@ def test_exact_cache_key_still_includes_prompt():
assert key_a != key_b
_TEAM_A_KEY_1: Final = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-a"}
_TEAM_A_KEY_2: Final = {"user_api_key_hash": "hash-2", "user_api_key_team_id": "team-a"}
_TEAM_B_KEY_3: Final = {"user_api_key_hash": "hash-3", "user_api_key_team_id": "team-b"}
_NO_TEAM_KEY_4: Final = {"user_api_key_hash": "hash-4"}
_NO_TEAM_KEY_5: Final = {"user_api_key_hash": "hash-5"}
def _scoped_keys(cache_scope: str | None, *identities: dict[str, str]) -> list[str]:
cache: Final = Cache(type=LiteLLMCacheType.LOCAL, **({} if cache_scope is None else {"cache_scope": cache_scope}))
messages: Final = [{"role": "user", "content": "What color is the sky?"}]
return [cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata=dict(i)) for i in identities]
@pytest.mark.parametrize(
("cache_scope", "same_team_shares", "other_team_shares", "teamless_keys_share"),
[
(None, True, True, True),
("global", True, True, True),
("team", True, False, False),
("key", False, False, False),
],
)
def test_cache_scope_controls_who_shares_an_exact_cache_hit(
cache_scope, same_team_shares, other_team_shares, teamless_keys_share
):
key_1, key_2, key_3, key_4, key_5 = _scoped_keys(
cache_scope, _TEAM_A_KEY_1, _TEAM_A_KEY_2, _TEAM_B_KEY_3, _NO_TEAM_KEY_4, _NO_TEAM_KEY_5
)
assert (key_1 == key_2) is same_team_shares
assert (key_1 == key_3) is other_team_shares
assert (key_4 == key_5) is teamless_keys_share
@pytest.mark.parametrize("cache_scope", ["team", "key"])
def test_cache_scope_is_stable_for_the_same_caller(cache_scope):
first, again = _scoped_keys(cache_scope, _TEAM_A_KEY_1, _TEAM_A_KEY_1)
assert first == again
def test_cache_scope_reads_identity_from_litellm_metadata_and_litellm_params():
cache: Final = Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team")
messages: Final = [{"role": "user", "content": "hi"}]
from_metadata: Final = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata=dict(_TEAM_A_KEY_1))
from_litellm_metadata: Final = cache.get_cache_key(
model="gpt-4o-mini", messages=messages, litellm_metadata=dict(_TEAM_A_KEY_1)
)
from_litellm_params: Final = cache.get_cache_key(
model="gpt-4o-mini", messages=messages, litellm_params={"metadata": dict(_TEAM_A_KEY_1)}
)
other_team: Final = cache.get_cache_key(
model="gpt-4o-mini", messages=messages, litellm_metadata=dict(_TEAM_B_KEY_3)
)
assert from_metadata == from_litellm_metadata == from_litellm_params
assert from_metadata != other_team
def test_cache_scope_rejects_unknown_value():
with pytest.raises(ValueError, match="'org' is not a valid CacheScope"):
Cache(type=LiteLLMCacheType.LOCAL, cache_scope="org")
@pytest.mark.asyncio
async def test_team_cache_scope_isolates_acompletion_hits_between_teams(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team"))
messages: Final = [{"role": "user", "content": f"team scope {uuid.uuid4().hex}"}]
async def call(identity: dict[str, str], mock_response: str):
return await acompletion(
model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response=mock_response
)
team_a_first: Final = await call(_TEAM_A_KEY_1, "a1")
await asyncio.gather(*_PENDING_CACHE_WRITES)
team_a_other_key: Final = await call(_TEAM_A_KEY_2, "a2")
team_b: Final = await call(_TEAM_B_KEY_3, "b")
assert team_a_other_key.id == team_a_first.id
assert team_b.id != team_a_first.id
assert team_b.choices[0].message.content == "b"
_NO_IDENTITY: Final = {}
_TEAMLESS_JWT_CALLER: Final = {"user_api_key_user_id": "jwt-user"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("cache_scope", "identity"),
[("team", _NO_IDENTITY), ("key", _NO_IDENTITY), ("team", _TEAMLESS_JWT_CALLER), ("key", _TEAMLESS_JWT_CALLER)],
)
async def test_scoped_cache_never_shares_hits_between_callers_without_identity(
monkeypatch: pytest.MonkeyPatch, cache_scope: str, identity: dict[str, str]
) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope))
messages: Final = [{"role": "user", "content": f"no identity {uuid.uuid4().hex}"}]
first: Final = await acompletion(model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response="1")
await asyncio.gather(*_PENDING_CACHE_WRITES)
second: Final = await acompletion(
model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response="2"
)
assert second.id != first.id
assert second.choices[0].message.content == "2"
@pytest.mark.asyncio
async def test_scoped_cache_without_identity_does_not_read_a_scoped_callers_entry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team"))
messages: Final = [{"role": "user", "content": f"cross read {uuid.uuid4().hex}"}]
team_a: Final = await acompletion(
model="gpt-4o-mini", messages=messages, metadata=dict(_TEAM_A_KEY_1), mock_response="a"
)
await asyncio.gather(*_PENDING_CACHE_WRITES)
anonymous: Final = await acompletion(model="gpt-4o-mini", messages=messages, mock_response="anon")
assert anonymous.id != team_a.id
@pytest.mark.asyncio
async def test_team_cache_scope_still_serves_cached_embeddings_to_the_same_team(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team"))
text: Final = f"embed {uuid.uuid4().hex}"
await aembedding(model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_A_KEY_1), mock_response=[0.1])
await asyncio.gather(*_PENDING_CACHE_WRITES)
same_team: Final = await aembedding(
model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_A_KEY_2), mock_response=[0.9]
)
other_team: Final = await aembedding(
model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_B_KEY_3), mock_response=[0.5]
)
assert same_team.data[0]["embedding"] == [0.1]
assert other_team.data[0]["embedding"] == [0.5]
@pytest.mark.parametrize(
"anthropic_param",
[

View file

@ -3,6 +3,7 @@ import copy
import json
import os
import time
from collections.abc import Mapping
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Final
@ -36,6 +37,8 @@ from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_log
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth
from litellm.caching.caching import Cache
from litellm.types.caching import LiteLLMCacheType
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES
from litellm.proxy.litellm_pre_call_utils import (
KeyAndTeamLoggingSettings,
@ -1228,6 +1231,69 @@ async def test_add_litellm_data_to_request_keeps_the_request_max_fallbacks_cap()
assert updated["max_fallbacks"] == 0
_FORGED_CACHE_KEY_FIELDS: Final = {
"cache_key": "ns:victim-hash",
"preset_cache_key": "ns:victim-hash",
"litellm_params": {
"preset_cache_key": "ns:victim-hash",
"metadata": {"user_api_key_team_id": "victim-team", "user_api_key_hash": "victim-key"},
},
}
_FORGED_TOP_LEVEL_CACHE_KEY_ONLY: Final = {"cache_key": "ns:victim-hash", "preset_cache_key": "ns:victim-hash"}
async def _add_data_with_forged_cache_key(
monkeypatch: pytest.MonkeyPatch, cache_scope: str, forged: Mapping[str, object] = _FORGED_CACHE_KEY_FIELDS
) -> dict:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope))
return await add_litellm_data_to_request(
data={
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hi"}],
**copy.deepcopy(dict(forged)),
},
request=_make_request_mock("/v1/chat/completions", {"Content-Type": "application/json"}),
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", team_id="caller-team"),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("cache_scope", ["team", "key"])
async def test_scoped_cache_ignores_caller_supplied_cache_keys(monkeypatch: pytest.MonkeyPatch, cache_scope: str):
updated: Final = await _add_data_with_forged_cache_key(monkeypatch, cache_scope)
assert "cache_key" not in updated
assert "preset_cache_key" not in updated
assert "preset_cache_key" not in updated["litellm_params"]
assert "user_api_key_team_id" not in updated["litellm_params"]["metadata"]
assert "user_api_key_hash" not in updated["litellm_params"]["metadata"]
assert updated["metadata"]["user_api_key_team_id"] == "caller-team"
@pytest.mark.asyncio
@pytest.mark.parametrize("cache_scope", ["team", "key"])
async def test_scoped_cache_strips_a_forged_cache_key_on_a_request_without_litellm_params(
monkeypatch: pytest.MonkeyPatch, cache_scope: str
):
updated: Final = await _add_data_with_forged_cache_key(monkeypatch, cache_scope, _FORGED_TOP_LEVEL_CACHE_KEY_ONLY)
assert "cache_key" not in updated
assert "preset_cache_key" not in updated
assert "litellm_params" not in updated
@pytest.mark.asyncio
async def test_global_cache_scope_keeps_caller_supplied_cache_key(monkeypatch: pytest.MonkeyPatch):
updated: Final = await _add_data_with_forged_cache_key(monkeypatch, "global")
assert updated["cache_key"] == "ns:victim-hash"
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_strips_user_control_fields():
"""Strip untrusted proxy-control fields before guardrails, logging, and headers read metadata."""

View file

@ -241,6 +241,20 @@ export const CACHE_FIELDS: readonly CacheField[] = [
helpText: "Namespace prefix for cache keys",
redisType: null,
},
{
name: "cache_scope",
label: "Cache Scope",
type: "select",
section: "cacheManagement",
helpText:
"Who can share a cache hit. Global shares hits between every caller. Team isolates hits per team, and keys without a team per key. Key isolates hits per key.",
redisType: null,
options: [
{ value: "global", label: "Global (shared by every caller)" },
{ value: "team", label: "Team (isolated per team)" },
{ value: "key", label: "Key (isolated per key)" },
],
},
{
name: "ttl",
label: "TTL (seconds)",

View file

@ -83,6 +83,16 @@ describe("buildCachePayload", () => {
expect(node).not.toHaveProperty("semantic_cache_scope");
});
it("should send the cache scope for every redis type and leave it to the server default when unset", () => {
const redisTypes = ["node", "cluster", "sentinel", "semantic"] as const;
for (const redisType of redisTypes) {
const scoped = buildCachePayload(redisType, { cache_scope: "team" }, { forTesting: false });
expect(scoped.cache_scope).toBe("team");
}
const unset = buildCachePayload("node", buildInitialValues({}), { forTesting: false });
expect(unset).not.toHaveProperty("cache_scope");
});
it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => {
const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true });
expect(payload.type).toBe("redis");