Merge pull request #1922 from BerriAI/litellm_bedrock_proxy_timeout

[FEAT] Set timeout for bedrock on proxy

litellm/proxy/proxy_config.yaml

@@ -29,6 +29,7 @@ model_list:
   - model_name: BEDROCK_GROUP
     litellm_params:
       model: bedrock/cohere.command-text-v14
+      timeout: 0.0001
   - model_name: tg-ai
     litellm_params:
       model: together_ai/mistralai/Mistral-7B-Instruct-v0.1

litellm/router.py

@@ -389,7 +389,7 @@ class Router:
             model_name = data["model"]
             for k, v in self.default_litellm_params.items():
                 if (
-                    k not in kwargs
+                    k not in kwargs and v is not None
                 ):  # prioritize model-specific params > default router params
                     kwargs[k] = v
                 elif k == "metadata":
@@ -409,13 +409,24 @@ class Router:
             else:
                 model_client = potential_model_client
             self.total_calls[model_name] += 1
+
+            timeout = (
+                data.get(
+                    "timeout", None
+                )  # timeout set on litellm_params for this deployment
+                or self.timeout  # timeout set on router
+                or kwargs.get(
+                    "timeout", None
+                )  # this uses default_litellm_params when nothing is set
+            )
+
             response = await litellm.acompletion(
                 **{
                     **data,
                     "messages": messages,
                     "caching": self.cache_responses,
                     "client": model_client,
-                    "timeout": self.timeout,
+                    "timeout": timeout,
                     **kwargs,
                 }
             )

litellm/tests/test_router_timeout.py

@@ -85,3 +85,47 @@ def test_router_timeouts():
 
         print("Response:", response)
         print("********** TOKENS USED SO FAR = ", total_tokens_used)
+
+
+@pytest.mark.asyncio
+async def test_router_timeouts_bedrock():
+    import openai
+
+    # Model list for OpenAI and Anthropic models
+    model_list = [
+        {
+            "model_name": "bedrock",
+            "litellm_params": {
+                "model": "bedrock/anthropic.claude-instant-v1",
+                "timeout": 0.001,
+            },
+            "tpm": 80000,
+        },
+    ]
+
+    # Configure router
+    router = Router(
+        model_list=model_list,
+        routing_strategy="usage-based-routing",
+        debug_level="DEBUG",
+        set_verbose=True,
+    )
+
+    litellm.set_verbose = True
+    try:
+        response = await router.acompletion(
+            model="bedrock",
+            messages=[{"role": "user", "content": "hello, who are u"}],
+        )
+        print(response)
+        pytest.fail("Did not raise error `openai.APITimeoutError`")
+    except openai.APITimeoutError as e:
+        print(
+            "Passed: Raised correct exception. Got openai.APITimeoutError\nGood Job", e
+        )
+        print(type(e))
+        pass
+    except Exception as e:
+        pytest.fail(
+            f"Did not raise error `openai.APITimeoutError`. Instead raised error type: {type(e)}, Error: {e}"
+        )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.23.5"
+version = "1.23.6"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -69,7 +69,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.23.5"
+version = "1.23.6"
 version_files = [
     "pyproject.toml:^version"
 ]