diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index aab62dd5db5..c8739c2ff2a 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -39,7 +39,7 @@ Safe to enable globally: from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from functools import cache -from typing import TYPE_CHECKING, Final, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Final, Optional, cast from pydantic import BaseModel, TypeAdapter @@ -69,7 +69,6 @@ if TYPE_CHECKING: _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) -_EncryptionBoundary: TypeAlias = tuple[object, object] | tuple[object, object, str, frozenset[tuple[str, object]]] @dataclass(frozen=True, slots=True) @@ -79,6 +78,21 @@ class _ProviderRoute: api_base: str | None +@dataclass(frozen=True, slots=True) +class _EncryptionBoundary: + api_base: object + api_key: object + model: str | None + account_scope: frozenset[tuple[str, object]] + explicit_pair: bool + + def is_shared_with(self, other: "_EncryptionBoundary") -> bool: + if (self.api_base, self.api_key) != (other.api_base, other.api_key): + return False + both_explicit: Final = self.explicit_pair and other.explicit_pair + return both_explicit or (self.model, self.account_scope) == (other.model, other.account_scope) + + class EncryptedContentAffinityCheck(CustomLogger): """ Routes follow-up Responses API requests to the deployment that produced @@ -229,11 +243,11 @@ class EncryptedContentAffinityCheck(CustomLogger): """ The upstream encryption boundary a deployment's encrypted reasoning belongs to. The values are resolved the way the request resolves them, the deployment's own values winning over its - named credential's, without modifying the deployment. An explicit ``(api_base, api_key)`` - pair is the whole boundary. On a provider that serves one vendor's models, a deployment with - no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint - its Responses URL builder resolves, and the boundary also carries the model and the headers - that pick an account. + named credential's, without modifying the deployment. Two explicit ``(api_base, api_key)`` + pairs match on those values alone. On a provider that serves one vendor's models, a deployment + with no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint + its Responses URL builder resolves, and a match involving either also needs the same model and + the same headers that pick an account. Accepts plain dicts (the common case in ``healthy_deployments``) as well as ``LiteLLM_Params`` Pydantic instances. A stricter @@ -247,10 +261,18 @@ class EncryptedContentAffinityCheck(CustomLogger): effective_params: Final = with_named_credential(params) explicit_api_key: Final = effective_params.get("api_key") explicit_api_base: Final = effective_params.get("api_base") + provider_route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) + route: Final = ( + provider_route + if provider_route is not None and provider_route.provider in _SINGLE_VENDOR_PROVIDERS + else None + ) + account_scope: Final = EncryptedContentAffinityCheck._account_scope(effective_params) if explicit_api_key and explicit_api_base: - return (explicit_api_base, explicit_api_key) - route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) - if route is None or route.provider not in _SINGLE_VENDOR_PROVIDERS: + return _EncryptionBoundary( + explicit_api_base, explicit_api_key, route.model if route else None, account_scope, explicit_pair=True + ) + if route is None: return None api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route) if not api_key: @@ -260,7 +282,7 @@ class EncryptedContentAffinityCheck(CustomLogger): ) if not api_base: return None - return (api_base, api_key, route.model, EncryptedContentAffinityCheck._account_scope(effective_params)) + return _EncryptionBoundary(api_base, api_key, route.model, account_scope, explicit_pair=False) @staticmethod def _params_mapping(litellm_params: object) -> Mapping[str, object] | None: @@ -356,7 +378,10 @@ class EncryptedContentAffinityCheck(CustomLogger): if boundary is None: return [], originating matches: Final = [ - d for d in healthy_deployments if self._encryption_boundary_key(d.get("litellm_params", {})) == boundary + d + for d in healthy_deployments + if (candidate := self._encryption_boundary_key(d.get("litellm_params", {}))) is not None + and boundary.is_shared_with(candidate) ] return matches, originating @@ -384,8 +409,8 @@ class EncryptedContentAffinityCheck(CustomLogger): ) @cache - def target_boundaries() -> frozenset[_EncryptionBoundary]: - return frozenset( + def target_boundaries() -> tuple[_EncryptionBoundary, ...]: + return tuple( boundary for target in target_deployments if (boundary := EncryptedContentAffinityCheck._encryption_boundary_key(target.get("litellm_params"))) @@ -409,7 +434,9 @@ class EncryptedContentAffinityCheck(CustomLogger): if origin is not None else None ) - return origin_boundary is not None and origin_boundary in target_boundaries() + return origin_boundary is not None and any( + origin_boundary.is_shared_with(target_boundary) for target_boundary in target_boundaries() + ) def should_strip_input_item(item: Mapping[str, object]) -> bool: return not target_can_decrypt(EncryptedContentAffinityCheck._model_id_of_input_item(item)) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index edd3b6d6e1d..e14a00499ed 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1103,14 +1103,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance(): plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params) assert pydantic_key is not None - assert ( - pydantic_key - == plain_key - == ( - "https://mateo-resource.openai.azure.com", - "fake-azure-resource-key-a", - ) - ) + assert pydantic_key == plain_key def test_boundary_key_resolves_missing_values_from_named_credential(): @@ -1134,7 +1127,13 @@ def test_boundary_key_resolves_missing_values_from_named_credential(): ): boundary = EncryptedContentAffinityCheck._encryption_boundary_key({"litellm_credential_name": "account-a"}) - assert boundary == ("https://account-a.example.com", "credential-key-a") + assert boundary is not None + assert boundary == EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "https://account-a.example.com", "api_key": "credential-key-a"} + ) + assert boundary != EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "https://account-a.example.com", "api_key": "credential-key-b"} + ) @pytest.mark.parametrize( @@ -2988,6 +2987,42 @@ def test_hop_strip_keeps_accounts_that_sign_in_through_headers_apart( assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) +@pytest.mark.parametrize( + "environment_key, target_model, target_can_decrypt", + [ + pytest.param("shared-key", GATEWAY_BASE["model"], True, id="environment key equal to the explicit key"), + pytest.param("other-key", GATEWAY_BASE["model"], False, id="environment key other than the explicit key"), + pytest.param("shared-key", "openai/gpt-5.5-mini", False, id="equal keys serving another model"), + ], +) +def test_hop_strip_matches_an_explicit_key_and_the_environment_key_on_one_api_base( + monkeypatch: pytest.MonkeyPatch, environment_key: str, target_model: str, target_can_decrypt: bool +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setenv("OPENAI_API_KEY", environment_key) + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", {**GATEWAY_BASE, "api_key": "shared-key"}), + _default_endpoint_deployment("target", {**GATEWAY_BASE, "model": target_model}), + ] + ) + ctx: Final = RoutingContext(router) + reasoning: Final = _reasoning_input_minted_by("origin")[0] + original_reasoning: Final = dict(reasoning) + request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + ctx, request_input, None, (_deployment_of(ctx, "target"),), unmarked_origin=None + ) + + assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) + + class _SecretReadCounter(CustomSecretManager): def __init__(self) -> None: super().__init__(secret_manager_name="read-counter")