fix(router): match an explicit key and the environment key on one api_base

A deployment with an explicit api_base and api_key now shares a boundary with one on the same api_base that takes the same key from OPENAI_API_KEY, when the model and account headers also match. Two explicit pairs still match on base and key alone, as on main.
This commit is contained in:
mateo-berri 2026-10-31 17:11:30 -07:00
parent d96515e76a
commit c0a44d6319
2 changed files with 86 additions and 24 deletions

View file

@ -39,7 +39,7 @@ Safe to enable globally:
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, Final, Optional, TypeAlias, cast
from typing import TYPE_CHECKING, Final, Optional, cast
from pydantic import BaseModel, TypeAdapter
@ -69,7 +69,6 @@ if TYPE_CHECKING:
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"})
_SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value})
_EncryptionBoundary: TypeAlias = tuple[object, object] | tuple[object, object, str, frozenset[tuple[str, object]]]
@dataclass(frozen=True, slots=True)
@ -79,6 +78,21 @@ class _ProviderRoute:
api_base: str | None
@dataclass(frozen=True, slots=True)
class _EncryptionBoundary:
api_base: object
api_key: object
model: str | None
account_scope: frozenset[tuple[str, object]]
explicit_pair: bool
def is_shared_with(self, other: "_EncryptionBoundary") -> bool:
if (self.api_base, self.api_key) != (other.api_base, other.api_key):
return False
both_explicit: Final = self.explicit_pair and other.explicit_pair
return both_explicit or (self.model, self.account_scope) == (other.model, other.account_scope)
class EncryptedContentAffinityCheck(CustomLogger):
"""
Routes follow-up Responses API requests to the deployment that produced
@ -229,11 +243,11 @@ class EncryptedContentAffinityCheck(CustomLogger):
"""
The upstream encryption boundary a deployment's encrypted reasoning belongs to. The values
are resolved the way the request resolves them, the deployment's own values winning over its
named credential's, without modifying the deployment. An explicit ``(api_base, api_key)``
pair is the whole boundary. On a provider that serves one vendor's models, a deployment with
no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint
its Responses URL builder resolves, and the boundary also carries the model and the headers
that pick an account.
named credential's, without modifying the deployment. Two explicit ``(api_base, api_key)``
pairs match on those values alone. On a provider that serves one vendor's models, a deployment
with no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint
its Responses URL builder resolves, and a match involving either also needs the same model and
the same headers that pick an account.
Accepts plain dicts (the common case in ``healthy_deployments``) as
well as ``LiteLLM_Params`` Pydantic instances. A stricter
@ -247,10 +261,18 @@ class EncryptedContentAffinityCheck(CustomLogger):
effective_params: Final = with_named_credential(params)
explicit_api_key: Final = effective_params.get("api_key")
explicit_api_base: Final = effective_params.get("api_base")
provider_route: Final = EncryptedContentAffinityCheck._provider_route(effective_params)
route: Final = (
provider_route
if provider_route is not None and provider_route.provider in _SINGLE_VENDOR_PROVIDERS
else None
)
account_scope: Final = EncryptedContentAffinityCheck._account_scope(effective_params)
if explicit_api_key and explicit_api_base:
return (explicit_api_base, explicit_api_key)
route: Final = EncryptedContentAffinityCheck._provider_route(effective_params)
if route is None or route.provider not in _SINGLE_VENDOR_PROVIDERS:
return _EncryptionBoundary(
explicit_api_base, explicit_api_key, route.model if route else None, account_scope, explicit_pair=True
)
if route is None:
return None
api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route)
if not api_key:
@ -260,7 +282,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
)
if not api_base:
return None
return (api_base, api_key, route.model, EncryptedContentAffinityCheck._account_scope(effective_params))
return _EncryptionBoundary(api_base, api_key, route.model, account_scope, explicit_pair=False)
@staticmethod
def _params_mapping(litellm_params: object) -> Mapping[str, object] | None:
@ -356,7 +378,10 @@ class EncryptedContentAffinityCheck(CustomLogger):
if boundary is None:
return [], originating
matches: Final = [
d for d in healthy_deployments if self._encryption_boundary_key(d.get("litellm_params", {})) == boundary
d
for d in healthy_deployments
if (candidate := self._encryption_boundary_key(d.get("litellm_params", {}))) is not None
and boundary.is_shared_with(candidate)
]
return matches, originating
@ -384,8 +409,8 @@ class EncryptedContentAffinityCheck(CustomLogger):
)
@cache
def target_boundaries() -> frozenset[_EncryptionBoundary]:
return frozenset(
def target_boundaries() -> tuple[_EncryptionBoundary, ...]:
return tuple(
boundary
for target in target_deployments
if (boundary := EncryptedContentAffinityCheck._encryption_boundary_key(target.get("litellm_params")))
@ -409,7 +434,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
if origin is not None
else None
)
return origin_boundary is not None and origin_boundary in target_boundaries()
return origin_boundary is not None and any(
origin_boundary.is_shared_with(target_boundary) for target_boundary in target_boundaries()
)
def should_strip_input_item(item: Mapping[str, object]) -> bool:
return not target_can_decrypt(EncryptedContentAffinityCheck._model_id_of_input_item(item))

View file

@ -1103,14 +1103,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance():
plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params)
assert pydantic_key is not None
assert (
pydantic_key
== plain_key
== (
"https://mateo-resource.openai.azure.com",
"fake-azure-resource-key-a",
)
)
assert pydantic_key == plain_key
def test_boundary_key_resolves_missing_values_from_named_credential():
@ -1134,7 +1127,13 @@ def test_boundary_key_resolves_missing_values_from_named_credential():
):
boundary = EncryptedContentAffinityCheck._encryption_boundary_key({"litellm_credential_name": "account-a"})
assert boundary == ("https://account-a.example.com", "credential-key-a")
assert boundary is not None
assert boundary == EncryptedContentAffinityCheck._encryption_boundary_key(
{"api_base": "https://account-a.example.com", "api_key": "credential-key-a"}
)
assert boundary != EncryptedContentAffinityCheck._encryption_boundary_key(
{"api_base": "https://account-a.example.com", "api_key": "credential-key-b"}
)
@pytest.mark.parametrize(
@ -2988,6 +2987,42 @@ def test_hop_strip_keeps_accounts_that_sign_in_through_headers_apart(
assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING)
@pytest.mark.parametrize(
"environment_key, target_model, target_can_decrypt",
[
pytest.param("shared-key", GATEWAY_BASE["model"], True, id="environment key equal to the explicit key"),
pytest.param("other-key", GATEWAY_BASE["model"], False, id="environment key other than the explicit key"),
pytest.param("shared-key", "openai/gpt-5.5-mini", False, id="equal keys serving another model"),
],
)
def test_hop_strip_matches_an_explicit_key_and_the_environment_key_on_one_api_base(
monkeypatch: pytest.MonkeyPatch, environment_key: str, target_model: str, target_can_decrypt: bool
):
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
EncryptedContentAffinityCheck,
)
monkeypatch.setattr(litellm, "api_key", None)
monkeypatch.setattr(litellm, "openai_key", None)
monkeypatch.setenv("OPENAI_API_KEY", environment_key)
router: Final = litellm.Router(
model_list=[
_default_endpoint_deployment("origin", {**GATEWAY_BASE, "api_key": "shared-key"}),
_default_endpoint_deployment("target", {**GATEWAY_BASE, "model": target_model}),
]
)
ctx: Final = RoutingContext(router)
reasoning: Final = _reasoning_input_minted_by("origin")[0]
original_reasoning: Final = dict(reasoning)
request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}]
EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
ctx, request_input, None, (_deployment_of(ctx, "target"),), unmarked_origin=None
)
assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING)
class _SecretReadCounter(CustomSecretManager):
def __init__(self) -> None:
super().__init__(secret_manager_name="read-counter")