Merge pull request #5352 from BerriAI/litellm_allow_setting_caching_mode

[Feat-Caching] allow setting caching mode to default off

docs/my-website/docs/proxy/caching.md

@@ -35,7 +35,7 @@ litellm_settings:
 
 #### [OPTIONAL] Step 1.5: Add redis namespaces, default ttl 
 
-## Namespace
+#### Namespace
 If you want to create some folder for your keys, you can set a namespace, like this:
 
 ```yaml
@@ -52,7 +52,7 @@ and keys will be stored like:
 litellm_caching:<hash>
 ```
 
-## Redis Cluster 
+#### Redis Cluster 
 
 ```yaml
 model_list:
@@ -68,7 +68,7 @@ litellm_settings:
     redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] 
 ```
 
-## TTL
+#### TTL
 
 ```yaml
 litellm_settings:
@@ -81,7 +81,7 @@ litellm_settings:
 ```
 
 
-## SSL
+#### SSL
 
 just set `REDIS_SSL="True"` in your .env, and LiteLLM will pick this up. 
 
@@ -397,7 +397,7 @@ litellm_settings:
                       # /chat/completions, /completions, /embeddings, /audio/transcriptions
 ```
 
-### Turn on / off caching per request.  
+### **Turn on / off caching per request. **
 
 The proxy support 4 cache-controls:
 
@@ -699,6 +699,73 @@ x-litellm-cache-key: 586bf3f3c1bf5aecb55bd9996494d3bbc69eb58397163add6d49537762a
              
 ```
 
+### **Set Caching Default Off - Opt in only **
+
+1. **Set `mode: default_off` for caching**
+
+```yaml
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+# default off mode
+litellm_settings:
+  set_verbose: True
+  cache: True
+  cache_params:
+    mode: default_off # 👈 Key change cache is default_off
+```
+
+2. **Opting in to cache when cache is default off**
+
+
+<Tabs>
+<TabItem value="openai" label="OpenAI Python SDK">
+
+```python
+import os
+from openai import OpenAI
+
+client = OpenAI(api_key=<litellm-api-key>, base_url="http://0.0.0.0:4000")
+
+chat_completion = client.chat.completions.create(
+    messages=[
+        {
+            "role": "user",
+            "content": "Say this is a test",
+        }
+    ],
+    model="gpt-3.5-turbo",
+    extra_body = {        # OpenAI python accepts extra args in extra_body
+        "cache": {"use-cache": True}
+    }
+)
+```
+</TabItem>
+
+<TabItem value="curl" label="curl">
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "cache": {"use-cache": True}
+    "messages": [
+      {"role": "user", "content": "Say this is a test"}
+    ]
+  }'
+```
+
+</TabItem>
+
+</Tabs>
+
+
 
 ### Turn on `batch_redis_requests` 
 

litellm/caching.py

@@ -16,6 +16,7 @@ import logging
 import time
 import traceback
 from datetime import timedelta
+from enum import Enum
 from typing import Any, BinaryIO, List, Literal, Optional, Union
 
 from openai._models import BaseModel as OpenAIObject
@@ -36,6 +37,11 @@ def print_verbose(print_statement):
         pass
 
 
+class CacheMode(str, Enum):
+    default_on = "default_on"
+    default_off = "default_off"
+
+
 class BaseCache:
     def set_cache(self, key, value, **kwargs):
         raise NotImplementedError
@@ -2079,6 +2085,9 @@ class Cache:
         type: Optional[
             Literal["local", "redis", "redis-semantic", "s3", "disk", "qdrant-semantic"]
         ] = "local",
+        mode: Optional[
+            CacheMode
+        ] = CacheMode.default_on,  # when default_on cache is always on, when default_off cache is opt in
         host: Optional[str] = None,
         port: Optional[str] = None,
         password: Optional[str] = None,
@@ -2214,6 +2223,7 @@ class Cache:
         self.namespace = namespace
         self.redis_flush_size = redis_flush_size
         self.ttl = ttl
+        self.mode: CacheMode = mode or CacheMode.default_on
 
         if self.type == "local" and default_in_memory_ttl is not None:
             self.ttl = default_in_memory_ttl
@@ -2420,6 +2430,8 @@ class Cache:
             The cached result if it exists, otherwise None.
         """
         try:  # never block execution
+            if self.should_use_cache(*args, **kwargs) is not True:
+                return
             messages = kwargs.get("messages", [])
             if "cache_key" in kwargs:
                 cache_key = kwargs["cache_key"]
@@ -2445,6 +2457,9 @@ class Cache:
         Used for embedding calls in async wrapper
         """
         try:  # never block execution
+            if self.should_use_cache(*args, **kwargs) is not True:
+                return
+
             messages = kwargs.get("messages", [])
             if "cache_key" in kwargs:
                 cache_key = kwargs["cache_key"]
@@ -2508,6 +2523,8 @@ class Cache:
             None
         """
         try:
+            if self.should_use_cache(*args, **kwargs) is not True:
+                return
             cache_key, cached_data, kwargs = self._add_cache_logic(
                 result=result, *args, **kwargs
             )
@@ -2521,6 +2538,8 @@ class Cache:
         Async implementation of add_cache
         """
         try:
+            if self.should_use_cache(*args, **kwargs) is not True:
+                return
             if self.type == "redis" and self.redis_flush_size is not None:
                 # high traffic - fill in results in memory and then flush
                 await self.batch_cache_write(result, *args, **kwargs)
@@ -2539,6 +2558,8 @@ class Cache:
         Does a bulk write, to prevent using too many clients
         """
         try:
+            if self.should_use_cache(*args, **kwargs) is not True:
+                return
             cache_list = []
             for idx, i in enumerate(kwargs["input"]):
                 preset_cache_key = self.get_cache_key(*args, **{**kwargs, "input": i})
@@ -2562,6 +2583,24 @@ class Cache:
         except Exception as e:
             verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
 
+    def should_use_cache(self, *args, **kwargs):
+        """
+        Returns true if we should use the cache for LLM API calls
+
+        If cache is default_on then this is True
+        If cache is default_off then this is only true when user has opted in to use cache
+        """
+        if self.mode == CacheMode.default_on:
+            return True
+
+        # when mode == default_off -> Cache is opt in only
+        _cache = kwargs.get("cache", None)
+        verbose_logger.debug("should_use_cache: kwargs: %s; _cache: %s", kwargs, _cache)
+        if _cache and isinstance(_cache, dict):
+            if _cache.get("use-cache", False) is True:
+                return True
+        return False
+
     async def batch_cache_write(self, result, *args, **kwargs):
         cache_key, cached_data, kwargs = self._add_cache_logic(
             result=result, *args, **kwargs

litellm/proxy/proxy_config.yaml

@@ -5,16 +5,9 @@ model_list:
       api_key: fake-key
       api_base: https://exampleopenaiendpoint-production.up.railway.app/
 
-guardrails:
-  - guardrail_name: "custom-pre-guard"
-    litellm_params:
-      guardrail: custom_guardrail.myCustomGuardrail  
-      mode: "pre_call"
-  - guardrail_name: "custom-during-guard"
-    litellm_params:
-      guardrail: custom_guardrail.myCustomGuardrail  
-      mode: "during_call"
-  - guardrail_name: "custom-post-guard"
-    litellm_params:
-      guardrail: custom_guardrail.myCustomGuardrail
-      mode: "post_call"
+# default off mode
+litellm_settings:
+  set_verbose: True
+  cache: True
+  cache_params:
+    mode: default_off

litellm/proxy/proxy_server.py

@@ -1604,7 +1604,7 @@ class ProxyConfig:
                     self._init_cache(cache_params=cache_params)
                     if litellm.cache is not None:
                         verbose_proxy_logger.debug(  # noqa
-                            f"{blue_color_code}Set Cache on LiteLLM Proxy: {vars(litellm.cache.cache)}{reset_color_code}"
+                            f"{blue_color_code}Set Cache on LiteLLM Proxy= {vars(litellm.cache.cache)}{vars(litellm.cache)}{reset_color_code}"
                         )
                 elif key == "cache" and value is False:
                     pass

litellm/tests/test_caching.py

@@ -1875,3 +1875,85 @@ async def test_qdrant_semantic_cache_acompletion_stream():
     except Exception as e:
         print(f"{str(e)}\n\n{traceback.format_exc()}")
         raise e
+
+
+@pytest.mark.asyncio()
+async def test_cache_default_off_acompletion():
+    litellm.set_verbose = True
+    import logging
+
+    from litellm._logging import verbose_logger
+
+    verbose_logger.setLevel(logging.DEBUG)
+
+    from litellm.caching import CacheMode
+
+    random_number = random.randint(
+        1, 100000
+    )  # add a random number to ensure it's always adding /reading from cache
+    litellm.cache = Cache(
+        type="local",
+        mode=CacheMode.default_off,
+    )
+
+    ### No Cache hits when it's default off
+
+    response1 = await litellm.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[
+            {
+                "role": "user",
+                "content": f"write a one sentence poem about: {random_number}",
+            }
+        ],
+        mock_response="hello",
+        max_tokens=20,
+    )
+    print(f"Response1: {response1}")
+
+    response2 = await litellm.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[
+            {
+                "role": "user",
+                "content": f"write a one sentence poem about: {random_number}",
+            }
+        ],
+        max_tokens=20,
+    )
+    print(f"Response2: {response2}")
+    assert response1.id != response2.id
+
+    ## Cache hits when it's default off and then opt in
+
+    response3 = await litellm.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[
+            {
+                "role": "user",
+                "content": f"write a one sentence poem about: {random_number}",
+            }
+        ],
+        mock_response="hello",
+        cache={"use-cache": True},
+        metadata={"key": "value"},
+        max_tokens=20,
+    )
+    print(f"Response3: {response3}")
+
+    await asyncio.sleep(2)
+
+    response4 = await litellm.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[
+            {
+                "role": "user",
+                "content": f"write a one sentence poem about: {random_number}",
+            }
+        ],
+        cache={"use-cache": True},
+        metadata={"key": "value"},
+        max_tokens=20,
+    )
+    print(f"Response4: {response4}")
+    assert response3.id == response4.id