Merge branch 'main' into litellm_metadata_logging_on_langfuse

docs/my-website/docs/proxy/deploy.md

@@ -1048,3 +1048,4 @@ export DATABASE_SCHEMA="schema-name" # skip to use the default "public" schema
 ```bash
 litellm --config /path/to/config.yaml --iam_token_db_auth
 ```
+

docs/my-website/docs/proxy/docker_quick_start.md

@@ -382,6 +382,56 @@ litellm_settings:
     ssl_verify: false # 👈 KEY CHANGE
 ```
 
+
+### (DB) All connection attempts failed 
+
+
+If you see:
+
+```
+httpx.ConnectError: All connection attempts failed                                                                        
+                                                                                                                         
+ERROR:    Application startup failed. Exiting.                                                                            
+3:21:43 - LiteLLM Proxy:ERROR: utils.py:2207 - Error getting LiteLLM_SpendLogs row count: All connection attempts failed 
+```
+
+This might be a DB permission issue. 
+
+1. Validate db user permission issue 
+
+Try creating a new database. 
+
+```bash
+STATEMENT: CREATE DATABASE "litellm"
+```
+
+If you get:
+
+```
+ERROR: permission denied to create 
+```
+
+This indicates you have a permission issue. 
+
+2. Grant permissions to your DB user
+
+It should look something like this:
+
+```
+psql -U postgres
+```
+
+```
+CREATE DATABASE litellm;
+```
+
+On CloudSQL, this is:
+
+```
+GRANT ALL PRIVILEGES ON DATABASE litellm TO your_username;
+```
+
+
 **What is `litellm_settings`?**
 
 LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing) for handling LLM API calls. 
@@ -398,3 +448,5 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing
 
 [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) 
 
+
+

docs/my-website/package-lock.json

@@ -21063,9 +21063,10 @@
       }
     },
     "node_modules/undici": {
-      "version": "6.21.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz",
-      "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==",
+      "version": "6.21.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.1.tgz",
+      "integrity": "sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==",
+      "license": "MIT",
       "engines": {
         "node": ">=18.17"
       }

tests/llm_translation/test_bedrock_completion.py

@@ -1659,6 +1659,7 @@ def test_bedrock_completion_test_3():
     ]
 
 
+@pytest.mark.skip(reason="Skipping this test as Bedrock now supports this behavior.")
 @pytest.mark.parametrize("modify_params", [True, False])
 def test_bedrock_completion_test_4(modify_params):
     litellm.set_verbose = True

tests/llm_translation/test_fireworks_ai_translation.py

@@ -93,6 +93,57 @@ class TestFireworksAIChatCompletion(BaseLLMChatTest):
         """
         pass
 
+    @pytest.mark.parametrize(
+        "response_format",
+        [
+            {"type": "json_object"},
+            {"type": "text"},
+        ],
+    )
+    @pytest.mark.flaky(retries=6, delay=1)
+    def test_json_response_format(self, response_format):
+        """
+        Test that the JSON response format is supported by the LLM API
+        """
+        from litellm.utils import supports_response_schema
+        from openai import OpenAI
+        from unittest.mock import patch
+
+        client = OpenAI()
+
+        base_completion_call_args = self.get_base_completion_call_args()
+        litellm.set_verbose = True
+
+        messages = [
+            {
+                "role": "system",
+                "content": "Your output should be a JSON object with no additional properties.  ",
+            },
+            {
+                "role": "user",
+                "content": "Respond with this in json. city=San Francisco, state=CA, weather=sunny, temp=60",
+            },
+        ]
+
+        with patch.object(
+            client.chat.completions.with_raw_response, "create"
+        ) as mock_post:
+            response = self.completion_function(
+                **base_completion_call_args,
+                messages=messages,
+                response_format=response_format,
+                client=client,
+            )
+
+            mock_post.assert_called_once()
+            if response_format["type"] == "json_object":
+                assert (
+                    mock_post.call_args.kwargs["response_format"]["type"]
+                    == "json_object"
+                )
+            else:
+                assert mock_post.call_args.kwargs["response_format"]["type"] == "text"
+
 
 class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest):
     def get_base_audio_transcription_call_args(self) -> dict:

tests/test_fallbacks.py

@@ -111,3 +111,53 @@ async def test_chat_completion_client_fallbacks(has_access):
         except Exception as e:
             if has_access:
                 pytest.fail("Expected this to work: {}".format(str(e)))
+
+
+@pytest.mark.parametrize("has_access", [True, False])
+@pytest.mark.asyncio
+async def test_chat_completion_client_fallbacks_with_custom_message(has_access):
+    """
+    make chat completion call with prompt > context window. expect it to work with fallback
+    """
+
+    async with aiohttp.ClientSession() as session:
+        models = ["gpt-3.5-turbo"]
+
+        if has_access:
+            models.append("gpt-instruct")
+
+        ## CREATE KEY WITH MODELS
+        generated_key = await generate_key(session=session, i=0, models=models)
+        calling_key = generated_key["key"]
+        model = "gpt-3.5-turbo"
+        messages = [
+            {"role": "user", "content": "Who was Alexander?"},
+        ]
+
+        ## CALL PROXY
+        try:
+            await chat_completion(
+                session=session,
+                key=calling_key,
+                model=model,
+                messages=messages,
+                mock_testing_fallbacks=True,
+                fallbacks=[
+                    {
+                        "model": "gpt-instruct",
+                        "messages": [
+                            {
+                                "role": "assistant",
+                                "content": "This is a custom message",
+                            }
+                        ],
+                    }
+                ],
+            )
+            if not has_access:
+                pytest.fail(
+                    "Expected this to fail, submitted fallback model that key did not have access to"
+                )
+        except Exception as e:
+            if has_access:
+                pytest.fail("Expected this to work: {}".format(str(e)))