From 5c88644b877bb5644c7c4e1e8ff01a729d6c386a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:56:23 -0800 Subject: [PATCH 1/6] build(deps): bump undici from 6.21.0 to 6.21.1 in /docs/my-website (#7902) Bumps [undici](https://github.com/nodejs/undici) from 6.21.0 to 6.21.1. - [Release notes](https://github.com/nodejs/undici/releases) - [Commits](https://github.com/nodejs/undici/compare/v6.21.0...v6.21.1) --- updated-dependencies: - dependency-name: undici dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/my-website/package-lock.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 527e0211eba..b5392b32b4f 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -21063,9 +21063,10 @@ } }, "node_modules/undici": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", - "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", + "version": "6.21.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.1.tgz", + "integrity": "sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==", + "license": "MIT", "engines": { "node": ">=18.17" } From 4e672f62691df5111e04aed0650dcd507018463d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 Jan 2025 18:19:49 -0800 Subject: [PATCH 2/6] fix: fix test --- .../logging_callback_tests/test_unit_tests_init_callbacks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 69bee7d402f..8b549f61800 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -193,8 +193,8 @@ async def use_callback_in_llm_call( elif used_in == "success_callback": print(f"litellm.success_callback: {litellm.success_callback}") print(f"litellm._async_success_callback: {litellm._async_success_callback}") - assert isinstance(litellm.success_callback[1], expected_class) - assert len(litellm.success_callback) == 2 # ["lago", LagoLogger] + assert isinstance(litellm.success_callback[0], expected_class) + assert len(litellm.success_callback) == 1 # [LagoLogger] assert isinstance(litellm._async_success_callback[0], expected_class) assert len(litellm._async_success_callback) == 1 From 55546f403b2b72c9662be2295e1d4da0e54886ab Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 Jan 2025 18:21:07 -0800 Subject: [PATCH 3/6] Revert "fix: fix test" This reverts commit 4e672f62691df5111e04aed0650dcd507018463d. --- .../logging_callback_tests/test_unit_tests_init_callbacks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 8b549f61800..69bee7d402f 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -193,8 +193,8 @@ async def use_callback_in_llm_call( elif used_in == "success_callback": print(f"litellm.success_callback: {litellm.success_callback}") print(f"litellm._async_success_callback: {litellm._async_success_callback}") - assert isinstance(litellm.success_callback[0], expected_class) - assert len(litellm.success_callback) == 1 # [LagoLogger] + assert isinstance(litellm.success_callback[1], expected_class) + assert len(litellm.success_callback) == 2 # ["lago", LagoLogger] assert isinstance(litellm._async_success_callback[0], expected_class) assert len(litellm._async_success_callback) == 1 From 049915c14bcea13666707a82c231db8a7eb46d5f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 Jan 2025 18:52:11 -0800 Subject: [PATCH 4/6] test: mock fireworks ai test - unstable api --- .../test_fireworks_ai_translation.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 5e1a1c05120..9e78270c922 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -93,6 +93,57 @@ class TestFireworksAIChatCompletion(BaseLLMChatTest): """ pass + @pytest.mark.parametrize( + "response_format", + [ + {"type": "json_object"}, + {"type": "text"}, + ], + ) + @pytest.mark.flaky(retries=6, delay=1) + def test_json_response_format(self, response_format): + """ + Test that the JSON response format is supported by the LLM API + """ + from litellm.utils import supports_response_schema + from openai import OpenAI + from unittest.mock import patch + + client = OpenAI() + + base_completion_call_args = self.get_base_completion_call_args() + litellm.set_verbose = True + + messages = [ + { + "role": "system", + "content": "Your output should be a JSON object with no additional properties. ", + }, + { + "role": "user", + "content": "Respond with this in json. city=San Francisco, state=CA, weather=sunny, temp=60", + }, + ] + + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_post: + response = self.completion_function( + **base_completion_call_args, + messages=messages, + response_format=response_format, + client=client, + ) + + mock_post.assert_called_once() + if response_format["type"] == "json_object": + assert ( + mock_post.call_args.kwargs["response_format"]["type"] + == "json_object" + ) + else: + assert mock_post.call_args.kwargs["response_format"]["type"] == "text" + class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest): def get_base_audio_transcription_call_args(self) -> dict: From 760ba4dfd5101381bdae6f365d7347318c9b1690 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 Jan 2025 18:53:12 -0800 Subject: [PATCH 5/6] test: skip test - Bedrock now supports this behavior --- tests/llm_translation/test_bedrock_completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 99dd207316a..1c5d560f982 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1659,6 +1659,7 @@ def test_bedrock_completion_test_3(): ] +@pytest.mark.skip(reason="Skipping this test as Bedrock now supports this behavior.") @pytest.mark.parametrize("modify_params", [True, False]) def test_bedrock_completion_test_4(modify_params): litellm.set_verbose = True From e3bacf71961abcbea5cc6e99c56eb48e470df3dd Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 22 Jan 2025 19:55:32 -0800 Subject: [PATCH 6/6] Litellm dev 01 22 2025 p1 (#7933) * docs(docker_quick_start.md): add more troubleshooting guides * test(test_fallbacks.py): add e2e test for proxy with fallbacks + custom fallback message * test(test_bedrock_completion.py): skip test now that bedrock supports this behaviour * test(test_fireworks_ai_translation.py): mock fireworks ai test --- docs/my-website/docs/proxy/deploy.md | 1 + .../docs/proxy/docker_quick_start.md | 52 +++++++++++++++++++ tests/test_fallbacks.py | 50 ++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 0be83725687..011778f5841 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -1048,3 +1048,4 @@ export DATABASE_SCHEMA="schema-name" # skip to use the default "public" schema ```bash litellm --config /path/to/config.yaml --iam_token_db_auth ``` + diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 9aa5ce15332..c5f28effa46 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -382,6 +382,56 @@ litellm_settings: ssl_verify: false # 👈 KEY CHANGE ``` + +### (DB) All connection attempts failed + + +If you see: + +``` +httpx.ConnectError: All connection attempts failed + +ERROR: Application startup failed. Exiting. +3:21:43 - LiteLLM Proxy:ERROR: utils.py:2207 - Error getting LiteLLM_SpendLogs row count: All connection attempts failed +``` + +This might be a DB permission issue. + +1. Validate db user permission issue + +Try creating a new database. + +```bash +STATEMENT: CREATE DATABASE "litellm" +``` + +If you get: + +``` +ERROR: permission denied to create +``` + +This indicates you have a permission issue. + +2. Grant permissions to your DB user + +It should look something like this: + +``` +psql -U postgres +``` + +``` +CREATE DATABASE litellm; +``` + +On CloudSQL, this is: + +``` +GRANT ALL PRIVILEGES ON DATABASE litellm TO your_username; +``` + + **What is `litellm_settings`?** LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing) for handling LLM API calls. @@ -398,3 +448,5 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) + + diff --git a/tests/test_fallbacks.py b/tests/test_fallbacks.py index e31761e1003..b2e27689ae9 100644 --- a/tests/test_fallbacks.py +++ b/tests/test_fallbacks.py @@ -111,3 +111,53 @@ async def test_chat_completion_client_fallbacks(has_access): except Exception as e: if has_access: pytest.fail("Expected this to work: {}".format(str(e))) + + +@pytest.mark.parametrize("has_access", [True, False]) +@pytest.mark.asyncio +async def test_chat_completion_client_fallbacks_with_custom_message(has_access): + """ + make chat completion call with prompt > context window. expect it to work with fallback + """ + + async with aiohttp.ClientSession() as session: + models = ["gpt-3.5-turbo"] + + if has_access: + models.append("gpt-instruct") + + ## CREATE KEY WITH MODELS + generated_key = await generate_key(session=session, i=0, models=models) + calling_key = generated_key["key"] + model = "gpt-3.5-turbo" + messages = [ + {"role": "user", "content": "Who was Alexander?"}, + ] + + ## CALL PROXY + try: + await chat_completion( + session=session, + key=calling_key, + model=model, + messages=messages, + mock_testing_fallbacks=True, + fallbacks=[ + { + "model": "gpt-instruct", + "messages": [ + { + "role": "assistant", + "content": "This is a custom message", + } + ], + } + ], + ) + if not has_access: + pytest.fail( + "Expected this to fail, submitted fallback model that key did not have access to" + ) + except Exception as e: + if has_access: + pytest.fail("Expected this to work: {}".format(str(e)))