From e170d66991ac2116f3e3a8cf0d291d5c148981e2 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 15 Dec 2023 14:16:34 +0530 Subject: [PATCH] docs - caching --- docs/my-website/docs/caching/redis_cache.md | 35 +++------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/docs/my-website/docs/caching/redis_cache.md b/docs/my-website/docs/caching/redis_cache.md index fac5aa2e6b2..979119ad7c8 100644 --- a/docs/my-website/docs/caching/redis_cache.md +++ b/docs/my-website/docs/caching/redis_cache.md @@ -55,32 +55,6 @@ litellm.cache = cache # set litellm.cache to your cache ``` -### Detecting Cached Responses -For resposes that were returned as cache hit, the response includes a param `cache` = True - -:::info - -Only valid for OpenAI <= 0.28.1 [Let us know if you still need this](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=bug&projects=&template=bug_report.yml&title=%5BBug%5D%3A+) -::: - -Example response with cache hit -```python -{ - 'cache': True, - 'id': 'chatcmpl-7wggdzd6OXhgE2YhcLJHJNZsEWzZ2', - 'created': 1694221467, - 'model': 'gpt-3.5-turbo-0613', - 'choices': [ - { - 'index': 0, 'message': {'role': 'assistant', 'content': 'I\'m sorry, but I couldn\'t find any information about "litellm" or how many stars it has. It is possible that you may be referring to a specific product, service, or platform that I am not familiar with. Can you please provide more context or clarify your question?' - }, 'finish_reason': 'stop'} - ], - 'usage': {'prompt_tokens': 17, 'completion_tokens': 59, 'total_tokens': 76}, -} - -``` - - ## Cache Initialization Parameters #### `type` (str, optional) @@ -124,15 +98,19 @@ Here's an example of accessing it: from litellm.integrations.custom_logger import CustomLogger from litellm import completion, acompletion, Cache +# create custom callback for success_events class MyCustomHandler(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): print(f"On Success") print(f"Value of Cache hit: {kwargs['cache_hit']"}) async def test_async_completion_azure_caching(): + # set custom callback customHandler_caching = MyCustomHandler() - litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD']) litellm.callbacks = [customHandler_caching] + + # init cache + litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD']) unique_time = time.time() response1 = await litellm.acompletion(model="azure/chatgpt-v-2", messages=[{ @@ -149,7 +127,4 @@ async def test_async_completion_azure_caching(): }], caching=True) await asyncio.sleep(1) # success callbacks are done in parallel - print(f"customHandler_caching.states post-cache hit: {customHandler_caching.states}") - assert len(customHandler_caching.errors) == 0 - assert len(customHandler_caching.states) == 4 # pre, post, success, success ```