fix(router): ask a secret manager for default boundaries at most once a minute per deployment

This commit is contained in:
mateo-berri 2026-11-01 13:17:23 -08:00
parent 252350e6d5
commit b29af5365f
2 changed files with 76 additions and 1 deletions

View file

@ -36,9 +36,11 @@ Safe to enable globally:
- No cache required.
"""
import hashlib
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from functools import cache, partial
from functools import cache, lru_cache, partial
from time import monotonic
from typing import TYPE_CHECKING, Final, Optional, cast
from pydantic import BaseModel, TypeAdapter, ValidationError
@ -58,6 +60,7 @@ from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployme
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
from litellm.secret_managers.main import secret_manager_would_be_consulted
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import Deployment, GenericLiteLLMParams
from litellm.types.utils import LlmProviders
@ -70,6 +73,8 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[object, object])
_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"})
_SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value})
_DEFAULT_ROUTE_SECRETS: Final = ("USE_LITELLM_PROXY", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE")
_SECRET_MANAGER_REFRESH_SECONDS: Final = 60
@dataclass(frozen=True, slots=True)
@ -95,6 +100,13 @@ class _EncryptionBoundary:
return self.account_scope == other.account_scope and self.model() == other.model()
@dataclass(frozen=True, slots=True)
class _ParamsInRefreshWindow:
digest: str
window: int
params: Mapping[str, object] = field(compare=False)
class EncryptedContentAffinityCheck(CustomLogger):
"""
Routes follow-up Responses API requests to the deployment that produced
@ -256,10 +268,30 @@ class EncryptedContentAffinityCheck(CustomLogger):
``isinstance(dict)`` guard would silently drop the latter from
boundary matching and fall back to the full pool — i.e. trigger the
exact ``invalid_encrypted_content`` failure this check exists to prevent.
When a secret manager would be asked for the provider defaults, a deployment's boundary is
reused for up to a minute, so a run of pin misses does not ask it again for every candidate.
"""
params: Final = EncryptedContentAffinityCheck._params_mapping(litellm_params)
if params is None:
return None
if not any(secret_manager_would_be_consulted(name) for name in _DEFAULT_ROUTE_SECRETS):
return EncryptedContentAffinityCheck._resolve_encryption_boundary(params)
return EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window(
_ParamsInRefreshWindow(
hashlib.sha256(repr(params).encode()).hexdigest(),
int(monotonic() // _SECRET_MANAGER_REFRESH_SECONDS),
params,
)
)
@staticmethod
@lru_cache(maxsize=1024)
def _encryption_boundary_in_refresh_window(key: _ParamsInRefreshWindow) -> _EncryptionBoundary | None:
return EncryptedContentAffinityCheck._resolve_encryption_boundary(key.params)
@staticmethod
def _resolve_encryption_boundary(params: Mapping[str, object]) -> _EncryptionBoundary | None:
effective_params: Final = with_named_credential(params)
explicit_api_key: Final = effective_params.get("api_key")
explicit_api_base: Final = effective_params.get("api_base")

View file

@ -3201,3 +3201,46 @@ async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_t
assert result == [peer]
assert reasoning_input[0] == original_reasoning
@pytest.mark.asyncio
async def test_repeated_pins_to_a_default_endpoint_peer_ask_the_secret_manager_once(monkeypatch: pytest.MonkeyPatch):
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
EncryptedContentAffinityCheck,
)
router: Final = litellm.Router(
model_list=[
_default_endpoint_deployment("origin", OPENAI_SHARED_KEY),
_default_endpoint_deployment("peer", OPENAI_SHARED_KEY),
_default_endpoint_deployment("other-org", {**OPENAI_SHARED_KEY, "api_key": "other-key"}),
]
)
ctx: Final = RoutingContext(router)
peer: Final = _deployment_of(ctx, "peer")
healthy_deployments: Final = [peer, _deployment_of(ctx, "other-org")]
secret_reads: Final = _SecretReadCounter()
monkeypatch.setattr(litellm, "secret_manager_client", secret_reads)
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM)
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only"))
EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window.cache_clear()
affinity: Final = EncryptedContentAffinityCheck(ctx=ctx)
async def pin_a_turn_minted_by_the_origin() -> tuple[list[dict[str, object]], Mapping[str, object]]:
reasoning_input: Final = _reasoning_input_minted_by("origin")
result: Final[list[dict[str, object]]] = await affinity.async_filter_deployments(
model="gpt-5.5",
healthy_deployments=list(healthy_deployments),
messages=None,
request_kwargs={"litellm_metadata": {}, "input": reasoning_input},
)
return result, reasoning_input[0]
first_pin, first_reasoning = await pin_a_turn_minted_by_the_origin()
reads_for_the_first_pin: Final = secret_reads.reads
second_pin, second_reasoning = await pin_a_turn_minted_by_the_origin()
assert first_pin == second_pin == [peer]
assert first_reasoning == second_reasoning == _reasoning_input_minted_by("origin")[0]
assert reads_for_the_first_pin > 0
assert secret_reads.reads == reads_for_the_first_pin