From 3c1da2e8230983258cf666000ac257bb4c452d8e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 24 Aug 2024 09:03:59 -0700 Subject: [PATCH 1/4] feat - allow setting cache mode --- litellm/caching.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/litellm/caching.py b/litellm/caching.py index 1b19fdf3e56..9b6011bf49d 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -16,6 +16,7 @@ import logging import time import traceback from datetime import timedelta +from enum import Enum from typing import Any, BinaryIO, List, Literal, Optional, Union from openai._models import BaseModel as OpenAIObject @@ -36,6 +37,11 @@ def print_verbose(print_statement): pass +class CacheMode(str, Enum): + default_on = "default_on" + default_off = "default_off" + + class BaseCache: def set_cache(self, key, value, **kwargs): raise NotImplementedError @@ -2079,6 +2085,9 @@ class Cache: type: Optional[ Literal["local", "redis", "redis-semantic", "s3", "disk", "qdrant-semantic"] ] = "local", + mode: Optional[ + CacheMode + ] = CacheMode.default_on, # when default_on cache is always on, when default_off cache is opt in host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, @@ -2214,6 +2223,7 @@ class Cache: self.namespace = namespace self.redis_flush_size = redis_flush_size self.ttl = ttl + self.mode: CacheMode = mode or CacheMode.default_on if self.type == "local" and default_in_memory_ttl is not None: self.ttl = default_in_memory_ttl @@ -2420,6 +2430,8 @@ class Cache: The cached result if it exists, otherwise None. """ try: # never block execution + if self.should_use_cache(*args, **kwargs) is not True: + return messages = kwargs.get("messages", []) if "cache_key" in kwargs: cache_key = kwargs["cache_key"] @@ -2445,6 +2457,9 @@ class Cache: Used for embedding calls in async wrapper """ try: # never block execution + if self.should_use_cache(*args, **kwargs) is not True: + return + messages = kwargs.get("messages", []) if "cache_key" in kwargs: cache_key = kwargs["cache_key"] @@ -2508,6 +2523,8 @@ class Cache: None """ try: + if self.should_use_cache(*args, **kwargs) is not True: + return cache_key, cached_data, kwargs = self._add_cache_logic( result=result, *args, **kwargs ) @@ -2521,6 +2538,8 @@ class Cache: Async implementation of add_cache """ try: + if self.should_use_cache(*args, **kwargs) is not True: + return if self.type == "redis" and self.redis_flush_size is not None: # high traffic - fill in results in memory and then flush await self.batch_cache_write(result, *args, **kwargs) @@ -2539,6 +2558,8 @@ class Cache: Does a bulk write, to prevent using too many clients """ try: + if self.should_use_cache(*args, **kwargs) is not True: + return cache_list = [] for idx, i in enumerate(kwargs["input"]): preset_cache_key = self.get_cache_key(*args, **{**kwargs, "input": i}) @@ -2562,6 +2583,20 @@ class Cache: except Exception as e: verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}") + def should_use_cache(self, *args, **kwargs): + if self.mode == CacheMode.default_on: + return True + else: + # when mode == default_off -> Cache is opt in only + _cache = kwargs.get("cache", None) + verbose_logger.debug( + f"should_use_cache: kwargs: {kwargs}; _cache: {_cache}" + ) + if _cache and isinstance(_cache, dict): + if _cache.get("use-cache", True) is False: + return True + return True + async def batch_cache_write(self, result, *args, **kwargs): cache_key, cached_data, kwargs = self._add_cache_logic( result=result, *args, **kwargs From 01adc45f0fe6b4ee3fc5d2e5c0304a89d1afc838 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 24 Aug 2024 09:06:59 -0700 Subject: [PATCH 2/4] test caching default on /off --- litellm/tests/test_caching.py | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index e474dff2e4a..402f3d51f1c 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1875,3 +1875,83 @@ async def test_qdrant_semantic_cache_acompletion_stream(): except Exception as e: print(f"{str(e)}\n\n{traceback.format_exc()}") raise e + + +@pytest.mark.asyncio() +async def test_cache_default_off_acompletion(): + litellm.set_verbose = True + import logging + + from litellm._logging import verbose_logger + + verbose_logger.setLevel(logging.DEBUG) + + from litellm.caching import CacheMode + + random_number = random.randint( + 1, 100000 + ) # add a random number to ensure it's always adding /reading from cache + litellm.cache = Cache( + type="local", + mode=CacheMode.default_off, + ) + + ### No Cache hits when it's default off + + response1 = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": f"write a one sentence poem about: {random_number}", + } + ], + mock_response="hello", + max_tokens=20, + ) + print(f"Response1: {response1}") + + response2 = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": f"write a one sentence poem about: {random_number}", + } + ], + max_tokens=20, + ) + print(f"Response2: {response2}") + assert response1.id != response2.id + + ## Cache hits when it's default off and then opt in + + response3 = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": f"write a one sentence poem about: {random_number}", + } + ], + mock_response="hello", + cache={"use-cache": True}, + metadata={"key": "value"}, + max_tokens=20, + ) + print(f"Response3: {response3}") + + response4 = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": f"write a one sentence poem about: {random_number}", + } + ], + cache={"use-cache": True}, + metadata={"key": "value"}, + max_tokens=20, + ) + print(f"Response4: {response4}") + assert response3.id == response4.id From feb354d3bc1e20fe376adcd670c64b2900b6381b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 24 Aug 2024 09:37:41 -0700 Subject: [PATCH 3/4] fix should_use_cache --- litellm/caching.py | 24 ++++++++++++++---------- litellm/tests/test_caching.py | 2 ++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 9b6011bf49d..880018c4c28 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -2584,18 +2584,22 @@ class Cache: verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}") def should_use_cache(self, *args, **kwargs): + """ + Returns true if we should use the cache for LLM API calls + + If cache is default_on then this is True + If cache is default_off then this is only true when user has opted in to use cache + """ if self.mode == CacheMode.default_on: return True - else: - # when mode == default_off -> Cache is opt in only - _cache = kwargs.get("cache", None) - verbose_logger.debug( - f"should_use_cache: kwargs: {kwargs}; _cache: {_cache}" - ) - if _cache and isinstance(_cache, dict): - if _cache.get("use-cache", True) is False: - return True - return True + + # when mode == default_off -> Cache is opt in only + _cache = kwargs.get("cache", None) + verbose_logger.debug("should_use_cache: kwargs: %s; _cache: %s", kwargs, _cache) + if _cache and isinstance(_cache, dict): + if _cache.get("use-cache", False) is True: + return True + return False async def batch_cache_write(self, result, *args, **kwargs): cache_key, cached_data, kwargs = self._add_cache_logic( diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 402f3d51f1c..34c161ba4b2 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1941,6 +1941,8 @@ async def test_cache_default_off_acompletion(): ) print(f"Response3: {response3}") + await asyncio.sleep(2) + response4 = await litellm.acompletion( model="gpt-3.5-turbo", messages=[ From 74f0e609627b80effa15775320c24ffbe203b923 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 24 Aug 2024 09:43:39 -0700 Subject: [PATCH 4/4] fix set Caching Default Off --- docs/my-website/docs/proxy/caching.md | 77 +++++++++++++++++++++++++-- litellm/proxy/proxy_config.yaml | 19 +++---- litellm/proxy/proxy_server.py | 2 +- 3 files changed, 79 insertions(+), 19 deletions(-) diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index dde2846fc1f..6324f424319 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -35,7 +35,7 @@ litellm_settings: #### [OPTIONAL] Step 1.5: Add redis namespaces, default ttl -## Namespace +#### Namespace If you want to create some folder for your keys, you can set a namespace, like this: ```yaml @@ -52,7 +52,7 @@ and keys will be stored like: litellm_caching: ``` -## Redis Cluster +#### Redis Cluster ```yaml model_list: @@ -68,7 +68,7 @@ litellm_settings: redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] ``` -## TTL +#### TTL ```yaml litellm_settings: @@ -81,7 +81,7 @@ litellm_settings: ``` -## SSL +#### SSL just set `REDIS_SSL="True"` in your .env, and LiteLLM will pick this up. @@ -397,7 +397,7 @@ litellm_settings: # /chat/completions, /completions, /embeddings, /audio/transcriptions ``` -### Turn on / off caching per request. +### **Turn on / off caching per request. ** The proxy support 4 cache-controls: @@ -699,6 +699,73 @@ x-litellm-cache-key: 586bf3f3c1bf5aecb55bd9996494d3bbc69eb58397163add6d49537762a ``` +### **Set Caching Default Off - Opt in only ** + +1. **Set `mode: default_off` for caching** + +```yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +# default off mode +litellm_settings: + set_verbose: True + cache: True + cache_params: + mode: default_off # 👈 Key change cache is default_off +``` + +2. **Opting in to cache when cache is default off** + + + + + +```python +import os +from openai import OpenAI + +client = OpenAI(api_key=, base_url="http://0.0.0.0:4000") + +chat_completion = client.chat.completions.create( + messages=[ + { + "role": "user", + "content": "Say this is a test", + } + ], + model="gpt-3.5-turbo", + extra_body = { # OpenAI python accepts extra args in extra_body + "cache": {"use-cache": True} + } +) +``` + + + + +```shell +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-3.5-turbo", + "cache": {"use-cache": True} + "messages": [ + {"role": "user", "content": "Say this is a test"} + ] + }' +``` + + + + + + ### Turn on `batch_redis_requests` diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 6be2454a2cd..ef11d798e5d 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -5,16 +5,9 @@ model_list: api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ -guardrails: - - guardrail_name: "custom-pre-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "pre_call" - - guardrail_name: "custom-during-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "during_call" - - guardrail_name: "custom-post-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "post_call" \ No newline at end of file +# default off mode +litellm_settings: + set_verbose: True + cache: True + cache_params: + mode: default_off diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5ccfc13c885..7e6f3c5e2fc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1604,7 +1604,7 @@ class ProxyConfig: self._init_cache(cache_params=cache_params) if litellm.cache is not None: verbose_proxy_logger.debug( # noqa - f"{blue_color_code}Set Cache on LiteLLM Proxy: {vars(litellm.cache.cache)}{reset_color_code}" + f"{blue_color_code}Set Cache on LiteLLM Proxy= {vars(litellm.cache.cache)}{vars(litellm.cache)}{reset_color_code}" ) elif key == "cache" and value is False: pass