diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 93d4a93bfe5..a679e73f94e 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -36,9 +36,11 @@ Safe to enable globally: - No cache required. """ +import hashlib from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, field -from functools import cache, partial +from functools import cache, lru_cache, partial +from time import monotonic from typing import TYPE_CHECKING, Final, Optional, cast from pydantic import BaseModel, TypeAdapter, ValidationError @@ -58,6 +60,7 @@ from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployme from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param +from litellm.secret_managers.main import secret_manager_would_be_consulted from litellm.types.llms.openai import AllMessageValues from litellm.types.router import Deployment, GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -70,6 +73,8 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _HEADERS_ADAPTER: Final = TypeAdapter(Mapping[object, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) +_DEFAULT_ROUTE_SECRETS: Final = ("USE_LITELLM_PROXY", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE") +_SECRET_MANAGER_REFRESH_SECONDS: Final = 60 @dataclass(frozen=True, slots=True) @@ -95,6 +100,13 @@ class _EncryptionBoundary: return self.account_scope == other.account_scope and self.model() == other.model() +@dataclass(frozen=True, slots=True) +class _ParamsInRefreshWindow: + digest: str + window: int + params: Mapping[str, object] = field(compare=False) + + class EncryptedContentAffinityCheck(CustomLogger): """ Routes follow-up Responses API requests to the deployment that produced @@ -256,10 +268,30 @@ class EncryptedContentAffinityCheck(CustomLogger): ``isinstance(dict)`` guard would silently drop the latter from boundary matching and fall back to the full pool — i.e. trigger the exact ``invalid_encrypted_content`` failure this check exists to prevent. + + When a secret manager would be asked for the provider defaults, a deployment's boundary is + reused for up to a minute, so a run of pin misses does not ask it again for every candidate. """ params: Final = EncryptedContentAffinityCheck._params_mapping(litellm_params) if params is None: return None + if not any(secret_manager_would_be_consulted(name) for name in _DEFAULT_ROUTE_SECRETS): + return EncryptedContentAffinityCheck._resolve_encryption_boundary(params) + return EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window( + _ParamsInRefreshWindow( + hashlib.sha256(repr(params).encode()).hexdigest(), + int(monotonic() // _SECRET_MANAGER_REFRESH_SECONDS), + params, + ) + ) + + @staticmethod + @lru_cache(maxsize=1024) + def _encryption_boundary_in_refresh_window(key: _ParamsInRefreshWindow) -> _EncryptionBoundary | None: + return EncryptedContentAffinityCheck._resolve_encryption_boundary(key.params) + + @staticmethod + def _resolve_encryption_boundary(params: Mapping[str, object]) -> _EncryptionBoundary | None: effective_params: Final = with_named_credential(params) explicit_api_key: Final = effective_params.get("api_key") explicit_api_base: Final = effective_params.get("api_base") diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index a36d320fe51..419318d8b72 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -3201,3 +3201,46 @@ async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_t assert result == [peer] assert reasoning_input[0] == original_reasoning + + +@pytest.mark.asyncio +async def test_repeated_pins_to_a_default_endpoint_peer_ask_the_secret_manager_once(monkeypatch: pytest.MonkeyPatch): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", OPENAI_SHARED_KEY), + _default_endpoint_deployment("peer", OPENAI_SHARED_KEY), + _default_endpoint_deployment("other-org", {**OPENAI_SHARED_KEY, "api_key": "other-key"}), + ] + ) + ctx: Final = RoutingContext(router) + peer: Final = _deployment_of(ctx, "peer") + healthy_deployments: Final = [peer, _deployment_of(ctx, "other-org")] + secret_reads: Final = _SecretReadCounter() + monkeypatch.setattr(litellm, "secret_manager_client", secret_reads) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window.cache_clear() + affinity: Final = EncryptedContentAffinityCheck(ctx=ctx) + + async def pin_a_turn_minted_by_the_origin() -> tuple[list[dict[str, object]], Mapping[str, object]]: + reasoning_input: Final = _reasoning_input_minted_by("origin") + result: Final[list[dict[str, object]]] = await affinity.async_filter_deployments( + model="gpt-5.5", + healthy_deployments=list(healthy_deployments), + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input}, + ) + return result, reasoning_input[0] + + first_pin, first_reasoning = await pin_a_turn_minted_by_the_origin() + reads_for_the_first_pin: Final = secret_reads.reads + second_pin, second_reasoning = await pin_a_turn_minted_by_the_origin() + + assert first_pin == second_pin == [peer] + assert first_reasoning == second_reasoning == _reasoning_input_minted_by("origin")[0] + assert reads_for_the_first_pin > 0 + assert secret_reads.reads == reads_for_the_first_pin