diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index df025949530..4710cb8c926 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1315,7 +1315,7 @@ class Logging: verbose_logger.debug( f"Model={self.model}; cost={self.model_call_details['response_cost']}" ) - except litellm.NotFoundError as e: + except litellm.NotFoundError: verbose_logger.warning( f"Model={self.model} not found in completion cost map. Setting 'response_cost' to None" ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b81f6abde9a..62faafe0e85 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,4 +1,15 @@ model_list: - - model_name: "ollama-llama3.1" + - model_name: gpt-3.5-turbo litellm_params: - model: "ollama_chat/llama3.1" \ No newline at end of file + model: gpt-3.5-turbo + +litellm_settings: + cache: True # set cache responses to True + cache_params: # set cache params for s3 + type: s3 + s3_bucket_name: litellm-proxy # AWS Bucket Name for S3 + s3_region_name: us-west-2 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 + + diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index ee8b0535401..89b83bcd664 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1033,11 +1033,10 @@ def test_disk_cache_completion(): assert response1.choices[0].message.content == response2.choices[0].message.content -@pytest.mark.skip(reason="AWS Suspended Account") +# @pytest.mark.skip(reason="AWS Suspended Account") +@pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio -async def test_s3_cache_acompletion_stream_azure(): - import asyncio - +async def test_s3_cache_stream_azure(sync_mode): try: litellm.set_verbose = True random_word = generate_random_word() @@ -1049,8 +1048,8 @@ async def test_s3_cache_acompletion_stream_azure(): ] litellm.cache = Cache( type="s3", - s3_bucket_name="litellm-my-test-bucket-2", - s3_region_name="us-east-1", + s3_bucket_name="litellm-proxy", + s3_region_name="us-west-2", ) print("s3 Cache: test for caching, streaming + completion") response_1_content = "" @@ -1059,34 +1058,65 @@ async def test_s3_cache_acompletion_stream_azure(): response_1_created = "" response_2_created = "" - response1 = await litellm.acompletion( - model="azure/chatgpt-v-2", - messages=messages, - max_tokens=40, - temperature=1, - stream=True, - ) - async for chunk in response1: - print(chunk) - response_1_created = chunk.created - response_1_content += chunk.choices[0].delta.content or "" - print(response_1_content) + if sync_mode: + response1 = litellm.completion( + model="azure/chatgpt-v-2", + messages=messages, + max_tokens=40, + temperature=1, + stream=True, + ) + for chunk in response1: + print(chunk) + response_1_created = chunk.created + response_1_content += chunk.choices[0].delta.content or "" + print(response_1_content) + else: + response1 = await litellm.acompletion( + model="azure/chatgpt-v-2", + messages=messages, + max_tokens=40, + temperature=1, + stream=True, + ) + async for chunk in response1: + print(chunk) + response_1_created = chunk.created + response_1_content += chunk.choices[0].delta.content or "" + print(response_1_content) - time.sleep(0.5) + if sync_mode: + time.sleep(0.5) + else: + await asyncio.sleep(0.5) print("\n\n Response 1 content: ", response_1_content, "\n\n") - response2 = await litellm.acompletion( - model="azure/chatgpt-v-2", - messages=messages, - max_tokens=40, - temperature=1, - stream=True, - ) - async for chunk in response2: - print(chunk) - response_2_content += chunk.choices[0].delta.content or "" - response_2_created = chunk.created - print(response_2_content) + if sync_mode: + response2 = litellm.completion( + model="azure/chatgpt-v-2", + messages=messages, + max_tokens=40, + temperature=1, + stream=True, + ) + for chunk in response2: + print(chunk) + response_2_content += chunk.choices[0].delta.content or "" + response_2_created = chunk.created + print(response_2_content) + else: + response2 = await litellm.acompletion( + model="azure/chatgpt-v-2", + messages=messages, + max_tokens=40, + temperature=1, + stream=True, + ) + async for chunk in response2: + print(chunk) + response_2_content += chunk.choices[0].delta.content or "" + response_2_created = chunk.created + print(response_2_content) print("\nresponse 1", response_1_content) print("\nresponse 2", response_2_content)