diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 0be83725687..011778f5841 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -1048,3 +1048,4 @@ export DATABASE_SCHEMA="schema-name" # skip to use the default "public" schema ```bash litellm --config /path/to/config.yaml --iam_token_db_auth ``` + diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 9aa5ce15332..c5f28effa46 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -382,6 +382,56 @@ litellm_settings: ssl_verify: false # 👈 KEY CHANGE ``` + +### (DB) All connection attempts failed + + +If you see: + +``` +httpx.ConnectError: All connection attempts failed + +ERROR: Application startup failed. Exiting. +3:21:43 - LiteLLM Proxy:ERROR: utils.py:2207 - Error getting LiteLLM_SpendLogs row count: All connection attempts failed +``` + +This might be a DB permission issue. + +1. Validate db user permission issue + +Try creating a new database. + +```bash +STATEMENT: CREATE DATABASE "litellm" +``` + +If you get: + +``` +ERROR: permission denied to create +``` + +This indicates you have a permission issue. + +2. Grant permissions to your DB user + +It should look something like this: + +``` +psql -U postgres +``` + +``` +CREATE DATABASE litellm; +``` + +On CloudSQL, this is: + +``` +GRANT ALL PRIVILEGES ON DATABASE litellm TO your_username; +``` + + **What is `litellm_settings`?** LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing) for handling LLM API calls. @@ -398,3 +448,5 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) + + diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 527e0211eba..b5392b32b4f 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -21063,9 +21063,10 @@ } }, "node_modules/undici": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", - "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", + "version": "6.21.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.1.tgz", + "integrity": "sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==", + "license": "MIT", "engines": { "node": ">=18.17" } diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 99dd207316a..1c5d560f982 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1659,6 +1659,7 @@ def test_bedrock_completion_test_3(): ] +@pytest.mark.skip(reason="Skipping this test as Bedrock now supports this behavior.") @pytest.mark.parametrize("modify_params", [True, False]) def test_bedrock_completion_test_4(modify_params): litellm.set_verbose = True diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 5e1a1c05120..9e78270c922 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -93,6 +93,57 @@ class TestFireworksAIChatCompletion(BaseLLMChatTest): """ pass + @pytest.mark.parametrize( + "response_format", + [ + {"type": "json_object"}, + {"type": "text"}, + ], + ) + @pytest.mark.flaky(retries=6, delay=1) + def test_json_response_format(self, response_format): + """ + Test that the JSON response format is supported by the LLM API + """ + from litellm.utils import supports_response_schema + from openai import OpenAI + from unittest.mock import patch + + client = OpenAI() + + base_completion_call_args = self.get_base_completion_call_args() + litellm.set_verbose = True + + messages = [ + { + "role": "system", + "content": "Your output should be a JSON object with no additional properties. ", + }, + { + "role": "user", + "content": "Respond with this in json. city=San Francisco, state=CA, weather=sunny, temp=60", + }, + ] + + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_post: + response = self.completion_function( + **base_completion_call_args, + messages=messages, + response_format=response_format, + client=client, + ) + + mock_post.assert_called_once() + if response_format["type"] == "json_object": + assert ( + mock_post.call_args.kwargs["response_format"]["type"] + == "json_object" + ) + else: + assert mock_post.call_args.kwargs["response_format"]["type"] == "text" + class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest): def get_base_audio_transcription_call_args(self) -> dict: diff --git a/tests/test_fallbacks.py b/tests/test_fallbacks.py index e31761e1003..b2e27689ae9 100644 --- a/tests/test_fallbacks.py +++ b/tests/test_fallbacks.py @@ -111,3 +111,53 @@ async def test_chat_completion_client_fallbacks(has_access): except Exception as e: if has_access: pytest.fail("Expected this to work: {}".format(str(e))) + + +@pytest.mark.parametrize("has_access", [True, False]) +@pytest.mark.asyncio +async def test_chat_completion_client_fallbacks_with_custom_message(has_access): + """ + make chat completion call with prompt > context window. expect it to work with fallback + """ + + async with aiohttp.ClientSession() as session: + models = ["gpt-3.5-turbo"] + + if has_access: + models.append("gpt-instruct") + + ## CREATE KEY WITH MODELS + generated_key = await generate_key(session=session, i=0, models=models) + calling_key = generated_key["key"] + model = "gpt-3.5-turbo" + messages = [ + {"role": "user", "content": "Who was Alexander?"}, + ] + + ## CALL PROXY + try: + await chat_completion( + session=session, + key=calling_key, + model=model, + messages=messages, + mock_testing_fallbacks=True, + fallbacks=[ + { + "model": "gpt-instruct", + "messages": [ + { + "role": "assistant", + "content": "This is a custom message", + } + ], + } + ], + ) + if not has_access: + pytest.fail( + "Expected this to fail, submitted fallback model that key did not have access to" + ) + except Exception as e: + if has_access: + pytest.fail("Expected this to work: {}".format(str(e)))