diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md
index 3f5342c7e60..b60b9966ba2 100644
--- a/docs/my-website/docs/proxy/caching.md
+++ b/docs/my-website/docs/proxy/caching.md
@@ -2,7 +2,6 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Caching
-Cache LLM Responses
:::note
@@ -10,14 +9,19 @@ For OpenAI/Anthropic Prompt Caching, go [here](../completion/prompt_caching.md)
:::
-LiteLLM supports:
+Cache LLM Responses. LiteLLM's caching system stores and reuses LLM responses to save costs and reduce latency. When you make the same request twice, the cached response is returned instead of calling the LLM API again.
+
+
+
+### Supported Caches
+
- In Memory Cache
- Redis Cache
- Qdrant Semantic Cache
- Redis Semantic Cache
- s3 Bucket Cache
-## Quick Start - Redis, s3 Cache, Semantic Cache
+## Quick Start
@@ -369,9 +373,9 @@ $ litellm --config /path/to/config.yaml
+## Usage
-
-## Using Caching - /chat/completions
+### Basic
@@ -416,6 +420,239 @@ curl --location 'http://0.0.0.0:4000/embeddings' \
+### Dynamic Cache Controls
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `ttl` | *Optional(int)* | Will cache the response for the user-defined amount of time (in seconds) |
+| `s-maxage` | *Optional(int)* | Will only accept cached responses that are within user-defined range (in seconds) |
+| `no-cache` | *Optional(bool)* | Will not store the response in cache. |
+| `no-store` | *Optional(bool)* | Will not cache the response |
+| `namespace` | *Optional(str)* | Will cache the response under a user-defined namespace |
+
+Each cache parameter can be controlled on a per-request basis. Here are examples for each parameter:
+
+### `ttl`
+
+Set how long (in seconds) to cache a response.
+
+
+
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="your-api-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+chat_completion = client.chat.completions.create(
+ messages=[{"role": "user", "content": "Hello"}],
+ model="gpt-3.5-turbo",
+ extra_body={
+ "cache": {
+ "ttl": 300 # Cache response for 5 minutes
+ }
+ }
+)
+```
+
+
+
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "cache": {"ttl": 300},
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ]
+ }'
+```
+
+
+
+### `s-maxage`
+
+Only accept cached responses that are within the specified age (in seconds).
+
+
+
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="your-api-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+chat_completion = client.chat.completions.create(
+ messages=[{"role": "user", "content": "Hello"}],
+ model="gpt-3.5-turbo",
+ extra_body={
+ "cache": {
+ "s-maxage": 600 # Only use cache if less than 10 minutes old
+ }
+ }
+)
+```
+
+
+
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "cache": {"s-maxage": 600},
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ]
+ }'
+```
+
+
+
+### `no-cache`
+Force a fresh response, bypassing the cache.
+
+
+
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="your-api-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+chat_completion = client.chat.completions.create(
+ messages=[{"role": "user", "content": "Hello"}],
+ model="gpt-3.5-turbo",
+ extra_body={
+ "cache": {
+ "no-cache": True # Skip cache check, get fresh response
+ }
+ }
+)
+```
+
+
+
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "cache": {"no-cache": true},
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ]
+ }'
+```
+
+
+
+### `no-store`
+
+Will not store the response in cache.
+
+
+
+
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="your-api-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+chat_completion = client.chat.completions.create(
+ messages=[{"role": "user", "content": "Hello"}],
+ model="gpt-3.5-turbo",
+ extra_body={
+ "cache": {
+ "no-store": True # Don't cache this response
+ }
+ }
+)
+```
+
+
+
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "cache": {"no-store": true},
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ]
+ }'
+```
+
+
+
+### `namespace`
+Store the response under a specific cache namespace.
+
+
+
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="your-api-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+chat_completion = client.chat.completions.create(
+ messages=[{"role": "user", "content": "Hello"}],
+ model="gpt-3.5-turbo",
+ extra_body={
+ "cache": {
+ "namespace": "my-custom-namespace" # Store in custom namespace
+ }
+ }
+)
+```
+
+
+
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "cache": {"namespace": "my-custom-namespace"},
+ "messages": [
+ {"role": "user", "content": "Hello"}
+ ]
+ }'
+```
+
+
+
+
+
## Set cache for proxy, but not on the actual llm api call
Use this if you just want to enable features like rate limiting, and loadbalancing across multiple instances.
@@ -501,253 +738,6 @@ litellm_settings:
# /chat/completions, /completions, /embeddings, /audio/transcriptions
```
-### **Turn on / off caching per request. **
-
-The proxy support 4 cache-controls:
-
-- `ttl`: *Optional(int)* - Will cache the response for the user-defined amount of time (in seconds).
-- `s-maxage`: *Optional(int)* Will only accept cached responses that are within user-defined range (in seconds).
-- `no-cache`: *Optional(bool)* Will not return a cached response, but instead call the actual endpoint.
-- `no-store`: *Optional(bool)* Will not cache the response.
-
-[Let us know if you need more](https://github.com/BerriAI/litellm/issues/1218)
-
-**Turn off caching**
-
-Set `no-cache=True`, this will not return a cached response
-
-
-
-
-```python
-import os
-from openai import OpenAI
-
-client = OpenAI(
- # This is the default and can be omitted
- api_key=os.environ.get("OPENAI_API_KEY"),
- base_url="http://0.0.0.0:4000"
-)
-
-chat_completion = client.chat.completions.create(
- messages=[
- {
- "role": "user",
- "content": "Say this is a test",
- }
- ],
- model="gpt-3.5-turbo",
- extra_body = { # OpenAI python accepts extra args in extra_body
- cache: {
- "no-cache": True # will not return a cached response
- }
- }
-)
-```
-
-
-
-
-```shell
-curl http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer sk-1234" \
- -d '{
- "model": "gpt-3.5-turbo",
- "cache": {"no-cache": True},
- "messages": [
- {"role": "user", "content": "Say this is a test"}
- ]
- }'
-```
-
-
-
-
-
-**Turn on caching**
-
-By default cache is always on
-
-
-
-
-```python
-import os
-from openai import OpenAI
-
-client = OpenAI(
- # This is the default and can be omitted
- api_key=os.environ.get("OPENAI_API_KEY"),
- base_url="http://0.0.0.0:4000"
-)
-
-chat_completion = client.chat.completions.create(
- messages=[
- {
- "role": "user",
- "content": "Say this is a test",
- }
- ],
- model="gpt-3.5-turbo"
-)
-```
-
-
-
-
-```shell
-curl http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer sk-1234" \
- -d '{
- "model": "gpt-3.5-turbo",
- "messages": [
- {"role": "user", "content": "Say this is a test"}
- ]
- }'
-```
-
-
-
-
-
-**Set `ttl`**
-
-Set `ttl=600`, this will caches response for 10 minutes (600 seconds)
-
-
-
-
-```python
-import os
-from openai import OpenAI
-
-client = OpenAI(
- # This is the default and can be omitted
- api_key=os.environ.get("OPENAI_API_KEY"),
- base_url="http://0.0.0.0:4000"
-)
-
-chat_completion = client.chat.completions.create(
- messages=[
- {
- "role": "user",
- "content": "Say this is a test",
- }
- ],
- model="gpt-3.5-turbo",
- extra_body = { # OpenAI python accepts extra args in extra_body
- cache: {
- "ttl": 600 # caches response for 10 minutes
- }
- }
-)
-```
-
-
-
-
-```shell
-curl http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer sk-1234" \
- -d '{
- "model": "gpt-3.5-turbo",
- "cache": {"ttl": 600},
- "messages": [
- {"role": "user", "content": "Say this is a test"}
- ]
- }'
-```
-
-
-
-
-
-
-
-**Set `s-maxage`**
-
-Set `s-maxage`, this will only get responses cached within last 10 minutes
-
-
-
-
-```python
-import os
-from openai import OpenAI
-
-client = OpenAI(
- # This is the default and can be omitted
- api_key=os.environ.get("OPENAI_API_KEY"),
- base_url="http://0.0.0.0:4000"
-)
-
-chat_completion = client.chat.completions.create(
- messages=[
- {
- "role": "user",
- "content": "Say this is a test",
- }
- ],
- model="gpt-3.5-turbo",
- extra_body = { # OpenAI python accepts extra args in extra_body
- cache: {
- "s-maxage": 600 # only get responses cached within last 10 minutes
- }
- }
-)
-```
-
-
-
-
-```shell
-curl http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer sk-1234" \
- -d '{
- "model": "gpt-3.5-turbo",
- "cache": {"s-maxage": 600},
- "messages": [
- {"role": "user", "content": "Say this is a test"}
- ]
- }'
-```
-
-
-
-
-
-
-### Turn on / off caching per Key.
-
-1. Add cache params when creating a key [full list](#turn-on--off-caching-per-key)
-
-```bash
-curl -X POST 'http://0.0.0.0:4000/key/generate' \
--H 'Authorization: Bearer sk-1234' \
--H 'Content-Type: application/json' \
--d '{
- "user_id": "222",
- "metadata": {
- "cache": {
- "no-cache": true
- }
- }
-}'
-```
-
-2. Test it!
-
-```bash
-curl -X POST 'http://localhost:4000/chat/completions' \
--H 'Content-Type: application/json' \
--H 'Authorization: Bearer ' \
--d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "bom dia"}]}'
-```
-
### Deleting Cache Keys - `/cache/delete`
In order to delete a cache key, send a request to `/cache/delete` with the `keys` you want to delete
diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py
index 90e37b07db6..26f94a94c29 100644
--- a/litellm/caching/caching.py
+++ b/litellm/caching/caching.py
@@ -277,9 +277,7 @@ class Cache:
verbose_logger.debug("\nCreated cache key: %s", cache_key)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
- hashed_cache_key = self._add_redis_namespace_to_cache_key(
- hashed_cache_key, **kwargs
- )
+ hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
self._set_preset_cache_key_in_kwargs(
preset_cache_key=hashed_cache_key, **kwargs
)
@@ -455,7 +453,7 @@ class Cache:
verbose_logger.debug("Hashed cache key (SHA-256): %s", hash_hex)
return hash_hex
- def _add_redis_namespace_to_cache_key(self, hash_hex: str, **kwargs) -> str:
+ def _add_namespace_to_cache_key(self, hash_hex: str, **kwargs) -> str:
"""
If a redis namespace is provided, add it to the cache key
@@ -466,7 +464,12 @@ class Cache:
Returns:
str: The final hashed cache key with the redis namespace.
"""
- namespace = kwargs.get("metadata", {}).get("redis_namespace") or self.namespace
+ dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {})
+ namespace = (
+ dynamic_cache_control.get("namespace")
+ or kwargs.get("metadata", {}).get("redis_namespace")
+ or self.namespace
+ )
if namespace:
hash_hex = f"{namespace}:{hash_hex}"
verbose_logger.debug("Final hashed key: %s", hash_hex)
@@ -546,11 +549,14 @@ class Cache:
else:
cache_key = self.get_cache_key(**kwargs)
if cache_key is not None:
- cache_control_args = kwargs.get("cache", {})
- max_age = cache_control_args.get(
- "s-max-age", cache_control_args.get("s-maxage", float("inf"))
+ cache_control_args: DynamicCacheControl = kwargs.get("cache", {})
+ max_age = (
+ cache_control_args.get("s-maxage")
+ or cache_control_args.get("s-max-age")
+ or float("inf")
)
cached_result = self.cache.get_cache(cache_key, messages=messages)
+ cached_result = self.cache.get_cache(cache_key, messages=messages)
return self._get_cache_logic(
cached_result=cached_result, max_age=max_age
)
diff --git a/litellm/types/caching.py b/litellm/types/caching.py
index a6f9de3084e..bc05f1bd8d0 100644
--- a/litellm/types/caching.py
+++ b/litellm/types/caching.py
@@ -33,3 +33,21 @@ class RedisPipelineIncrementOperation(TypedDict):
key: str
increment_value: float
ttl: Optional[int]
+
+
+DynamicCacheControl = TypedDict(
+ "DynamicCacheControl",
+ {
+ # Will cache the response for the user-defined amount of time (in seconds).
+ "ttl": Optional[int],
+ # Namespace to use for caching
+ "namespace": Optional[str],
+ # Max Age to use for caching
+ "s-maxage": Optional[int],
+ "s-max-age": Optional[int],
+ # Will not return a cached response, but instead call the actual endpoint.
+ "no-cache": Optional[bool],
+ # Will not store the response in the cache.
+ "no-store": Optional[bool],
+ },
+)
diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py
index 04110dae4e2..ae1e4d38c32 100644
--- a/tests/local_testing/test_caching.py
+++ b/tests/local_testing/test_caching.py
@@ -2478,3 +2478,47 @@ async def test_redis_get_ttl():
except Exception as e:
print(f"Error occurred: {str(e)}")
raise e
+
+
+def test_redis_caching_multiple_namespaces():
+ """
+ Test that redis caching works with multiple namespaces
+
+ If client side request specifies a namespace, it should be used for caching
+
+ The same request with different namespaces should not be cached under the same key
+ """
+ import uuid
+
+ messages = [{"role": "user", "content": f"what is litellm? {uuid.uuid4()}"}]
+ litellm.cache = Cache(type="redis")
+ namespace_1 = "org-id1"
+ namespace_2 = "org-id2"
+
+ response_1 = completion(
+ model="gpt-3.5-turbo", messages=messages, cache={"namespace": namespace_1}
+ )
+
+ response_2 = completion(
+ model="gpt-3.5-turbo", messages=messages, cache={"namespace": namespace_2}
+ )
+
+ response_3 = completion(
+ model="gpt-3.5-turbo", messages=messages, cache={"namespace": namespace_1}
+ )
+
+ response_4 = completion(model="gpt-3.5-turbo", messages=messages)
+
+ print("response 1: ", response_1.model_dump_json(indent=4))
+ print("response 2: ", response_2.model_dump_json(indent=4))
+ print("response 3: ", response_3.model_dump_json(indent=4))
+ print("response 4: ", response_4.model_dump_json(indent=4))
+
+ # request 1 & 3 used under the same namespace
+ assert response_1.id == response_3.id
+
+ # request 2 used under a different namespace
+ assert response_2.id != response_1.id
+
+ # request 4 without a namespace should not be cached under the same key as request 3
+ assert response_4.id != response_3.id
diff --git a/tests/local_testing/test_unit_test_caching.py b/tests/local_testing/test_unit_test_caching.py
index 52007698eef..b1e8d4fe61e 100644
--- a/tests/local_testing/test_unit_test_caching.py
+++ b/tests/local_testing/test_unit_test_caching.py
@@ -137,19 +137,28 @@ def test_get_hashed_cache_key():
assert len(hashed_key) == 64 # SHA-256 produces a 64-character hex string
-def test_add_redis_namespace_to_cache_key():
+def test_add_namespace_to_cache_key():
cache = Cache(namespace="test_namespace")
hashed_key = "abcdef1234567890"
# Test with class-level namespace
- result = cache._add_redis_namespace_to_cache_key(hashed_key)
+ result = cache._add_namespace_to_cache_key(hashed_key)
assert result == "test_namespace:abcdef1234567890"
# Test with metadata namespace
kwargs = {"metadata": {"redis_namespace": "custom_namespace"}}
- result = cache._add_redis_namespace_to_cache_key(hashed_key, **kwargs)
+ result = cache._add_namespace_to_cache_key(hashed_key, **kwargs)
assert result == "custom_namespace:abcdef1234567890"
+ # Test with cache control namespace
+ kwargs = {"cache": {"namespace": "cache_control_namespace"}}
+ result = cache._add_namespace_to_cache_key(hashed_key, **kwargs)
+ assert result == "cache_control_namespace:abcdef1234567890"
+
+ kwargs = {"cache": {"namespace": "cache_control_namespace-2"}}
+ result = cache._add_namespace_to_cache_key(hashed_key, **kwargs)
+ assert result == "cache_control_namespace-2:abcdef1234567890"
+
def test_get_model_param_value():
cache = Cache()