mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(router): ask a secret manager for default boundaries at most once a minute per deployment
This commit is contained in:
parent
252350e6d5
commit
b29af5365f
2 changed files with 76 additions and 1 deletions
|
|
@ -36,9 +36,11 @@ Safe to enable globally:
|
|||
- No cache required.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache, partial
|
||||
from functools import cache, lru_cache, partial
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Final, Optional, cast
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
|
|
@ -58,6 +60,7 @@ from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployme
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
|
||||
from litellm.secret_managers.main import secret_manager_would_be_consulted
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import Deployment, GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -70,6 +73,8 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
|||
_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[object, object])
|
||||
_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"})
|
||||
_SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value})
|
||||
_DEFAULT_ROUTE_SECRETS: Final = ("USE_LITELLM_PROXY", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE")
|
||||
_SECRET_MANAGER_REFRESH_SECONDS: Final = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -95,6 +100,13 @@ class _EncryptionBoundary:
|
|||
return self.account_scope == other.account_scope and self.model() == other.model()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ParamsInRefreshWindow:
|
||||
digest: str
|
||||
window: int
|
||||
params: Mapping[str, object] = field(compare=False)
|
||||
|
||||
|
||||
class EncryptedContentAffinityCheck(CustomLogger):
|
||||
"""
|
||||
Routes follow-up Responses API requests to the deployment that produced
|
||||
|
|
@ -256,10 +268,30 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
``isinstance(dict)`` guard would silently drop the latter from
|
||||
boundary matching and fall back to the full pool — i.e. trigger the
|
||||
exact ``invalid_encrypted_content`` failure this check exists to prevent.
|
||||
|
||||
When a secret manager would be asked for the provider defaults, a deployment's boundary is
|
||||
reused for up to a minute, so a run of pin misses does not ask it again for every candidate.
|
||||
"""
|
||||
params: Final = EncryptedContentAffinityCheck._params_mapping(litellm_params)
|
||||
if params is None:
|
||||
return None
|
||||
if not any(secret_manager_would_be_consulted(name) for name in _DEFAULT_ROUTE_SECRETS):
|
||||
return EncryptedContentAffinityCheck._resolve_encryption_boundary(params)
|
||||
return EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window(
|
||||
_ParamsInRefreshWindow(
|
||||
hashlib.sha256(repr(params).encode()).hexdigest(),
|
||||
int(monotonic() // _SECRET_MANAGER_REFRESH_SECONDS),
|
||||
params,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@lru_cache(maxsize=1024)
|
||||
def _encryption_boundary_in_refresh_window(key: _ParamsInRefreshWindow) -> _EncryptionBoundary | None:
|
||||
return EncryptedContentAffinityCheck._resolve_encryption_boundary(key.params)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_encryption_boundary(params: Mapping[str, object]) -> _EncryptionBoundary | None:
|
||||
effective_params: Final = with_named_credential(params)
|
||||
explicit_api_key: Final = effective_params.get("api_key")
|
||||
explicit_api_base: Final = effective_params.get("api_base")
|
||||
|
|
|
|||
|
|
@ -3201,3 +3201,46 @@ async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_t
|
|||
|
||||
assert result == [peer]
|
||||
assert reasoning_input[0] == original_reasoning
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_pins_to_a_default_endpoint_peer_ask_the_secret_manager_once(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_default_endpoint_deployment("origin", OPENAI_SHARED_KEY),
|
||||
_default_endpoint_deployment("peer", OPENAI_SHARED_KEY),
|
||||
_default_endpoint_deployment("other-org", {**OPENAI_SHARED_KEY, "api_key": "other-key"}),
|
||||
]
|
||||
)
|
||||
ctx: Final = RoutingContext(router)
|
||||
peer: Final = _deployment_of(ctx, "peer")
|
||||
healthy_deployments: Final = [peer, _deployment_of(ctx, "other-org")]
|
||||
secret_reads: Final = _SecretReadCounter()
|
||||
monkeypatch.setattr(litellm, "secret_manager_client", secret_reads)
|
||||
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM)
|
||||
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only"))
|
||||
EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window.cache_clear()
|
||||
affinity: Final = EncryptedContentAffinityCheck(ctx=ctx)
|
||||
|
||||
async def pin_a_turn_minted_by_the_origin() -> tuple[list[dict[str, object]], Mapping[str, object]]:
|
||||
reasoning_input: Final = _reasoning_input_minted_by("origin")
|
||||
result: Final[list[dict[str, object]]] = await affinity.async_filter_deployments(
|
||||
model="gpt-5.5",
|
||||
healthy_deployments=list(healthy_deployments),
|
||||
messages=None,
|
||||
request_kwargs={"litellm_metadata": {}, "input": reasoning_input},
|
||||
)
|
||||
return result, reasoning_input[0]
|
||||
|
||||
first_pin, first_reasoning = await pin_a_turn_minted_by_the_origin()
|
||||
reads_for_the_first_pin: Final = secret_reads.reads
|
||||
second_pin, second_reasoning = await pin_a_turn_minted_by_the_origin()
|
||||
|
||||
assert first_pin == second_pin == [peer]
|
||||
assert first_reasoning == second_reasoning == _reasoning_input_minted_by("origin")[0]
|
||||
assert reads_for_the_first_pin > 0
|
||||
assert secret_reads.reads == reads_for_the_first_pin
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue