Merge pull request #50217 from BerriAI/litellm_backport_team_passthrough_skip_lts_1108

fix(passthrough): skip team deployments when picking proxy-wide pass-through credentials on lts/1.108.x

litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py

@@ -1,5 +1,5 @@
 import json
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Final
 
@@ -11,6 +11,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
     LiteLLM_ManagedVectorStore,
 )
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
 from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
@@ -26,6 +27,11 @@ def _get_proxy_llm_router() -> "Router | None":
     return llm_router
 
 
+def _is_team_deployment(deployment: Mapping[str, object]) -> bool:
+    model_info: Final = deployment.get("model_info")
+    return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id"))
+
+
 def _get_str_value(values: dict[str, object] | None, key: str) -> str | None:
     value: Final = values.get(key) if values is not None else None
     return value if isinstance(value, str) else None
@@ -101,6 +107,8 @@ class PassthroughEndpointRouter:
         if llm_router is None:
             return None
         for deployment in llm_router.get_model_list() or ():
+            if _is_team_deployment(deployment):
+                continue
             litellm_params: LiteLLMParamsTypedDict = deployment["litellm_params"]
             api_key: str | None = self._resolve_matching_deployment_api_key(
                 litellm_params=litellm_params,

tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py

@@ -1,6 +1,7 @@
 import asyncio
 import os
 import unittest
+from typing import Final
 from unittest.mock import patch
 
 import pytest
@@ -60,6 +61,43 @@ def test_credential_loaded_after_deployment_registration_still_resolves():
     )
 
 
+def _team_deployment(model: str, team_id: str, **litellm_params: str) -> dict[str, object]:
+    return {**_flagged_deployment(model, **litellm_params), "model_info": {"team_id": team_id}}
+
+
+def test_proxy_wide_deployment_credentials_skip_a_team_deployment_for_the_proxy_wide_one() -> None:
+    llm_router: Final = litellm.Router(
+        model_list=[
+            _team_deployment(
+                "watsonx/ibm/granite-3-8b-instruct",
+                "team-a",
+                api_key="wx-team-a-key",
+                api_base="https://team-a.ml.cloud.ibm.com",
+            ),
+            _flagged_deployment(
+                "watsonx/ibm/granite-3-8b-instruct",
+                api_key="wx-proxy-wide-key",
+                api_base="https://eu-de.ml.cloud.ibm.com",
+            ),
+        ]
+    )
+
+    credentials: Final = _passthrough_router(llm_router).get_proxy_wide_deployment_credentials(
+        custom_llm_provider="watsonx"
+    )
+
+    assert credentials is not None
+    assert (credentials.api_key, credentials.api_base) == ("wx-proxy-wide-key", "https://eu-de.ml.cloud.ibm.com")
+
+
+def test_proxy_wide_deployment_credentials_are_none_when_only_a_team_deployment_is_flagged() -> None:
+    llm_router: Final = litellm.Router(
+        model_list=[_team_deployment("watsonx/ibm/granite-3-8b-instruct", "team-a", api_key="wx-team-a-key")]
+    )
+
+    assert _passthrough_router(llm_router).get_proxy_wide_deployment_credentials(custom_llm_provider="watsonx") is None
+
+
 def test_credential_rotation_is_reflected_without_deployment_update():
     CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-before-rotation")])
     llm_router = litellm.Router(

tests/unit/proxy/pass_through_endpoints/test_watsonx_proxy_route.py

@@ -6,6 +6,7 @@ and version parameter injection.
 """
 
 import json
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
 
 import pytest
@@ -554,3 +555,27 @@ async def test_watsonx_route_falls_back_to_env_when_no_deployment_is_flagged(mon
     assert target == "https://us-south.ml.cloud.ibm.com/ml/v1/text/generation"
     assert authorization == "Bearer iam-token-1"
     assert iam_apikey == "wx-env-key"
+
+
+@pytest.mark.asyncio
+async def test_watsonx_route_falls_back_to_env_when_the_only_flagged_deployment_belongs_to_a_team(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    llm_router: Final = litellm.Router(
+        model_list=[
+            {
+                **_watsonx_deployment("wx-team-a-key", "https://team-a.ml.cloud.ibm.com", use_in_pass_through=True),
+                "model_info": {"team_id": "team-a"},
+            }
+        ]
+    )
+
+    target, authorization, iam_apikey = await _route_target_and_iam_apikey(
+        monkeypatch,
+        llm_router,
+        env={"WX_API_KEY": "wx-env-key", "WATSONX_API_BASE": "https://us-south.ml.cloud.ibm.com"},
+    )
+
+    assert target == "https://us-south.ml.cloud.ibm.com/ml/v1/text/generation"
+    assert authorization == "Bearer iam-token-1"
+    assert iam_apikey == "wx-env-key"