fix(router): resolve default boundaries only for openai and drop the unsent organization param

Aggregators and Bedrock Mantle serve several vendors on one key, so a missing key or base there no longer yields a shared boundary. The Responses request never sends the organization param, so it no longer splits the boundary, while a Cookie header now does.
This commit is contained in:
mateo-berri 2026-10-31 15:14:57 -07:00
parent 9c3deef179
commit f9e5a09e7b
2 changed files with 22 additions and 30 deletions

View file

@ -66,8 +66,8 @@ if TYPE_CHECKING:
from litellm.router_api import RoutingContext
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project"})
_PROVIDER_NAMES: Final = frozenset(provider.value for provider in LlmProviders)
_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"})
_SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value})
@dataclass(frozen=True, slots=True)
@ -227,9 +227,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
"""
``(api_base, api_key, account scope)`` identifies an upstream encryption boundary.
The values are resolved the way the request resolves them, the deployment's own values
winning over its named credential's, without modifying the deployment. A deployment with
no ``api_key`` takes its provider's default key, and one with no ``api_base`` takes the
endpoint its provider's Responses URL builder resolves for it.
winning over its named credential's, without modifying the deployment. On a provider that
serves one vendor's models, a deployment with no ``api_key`` takes the provider's default
key, and one with no ``api_base`` takes the endpoint its Responses URL builder resolves.
Accepts plain dicts (the common case in ``healthy_deployments``) as
well as ``LiteLLM_Params`` Pydantic instances. A stricter
@ -247,7 +247,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
if explicit_api_key and explicit_api_base:
return (explicit_api_base, explicit_api_key, account_scope)
route: Final = EncryptedContentAffinityCheck._provider_route(effective_params)
if route is None:
if route is None or route.provider not in _SINGLE_VENDOR_PROVIDERS:
return None
api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route)
if not api_key:
@ -269,15 +269,11 @@ class EncryptedContentAffinityCheck(CustomLogger):
@staticmethod
def _account_scope(litellm_params: Mapping[str, object]) -> frozenset[tuple[str, object]]:
organization: Final = litellm_params.get("organization")
header_scope: Final = frozenset(
return frozenset(
(name.lower(), str(value))
for name, value in EncryptedContentAffinityCheck._header_items(litellm_params)
if name.lower() in _ACCOUNT_SCOPE_HEADERS or has_sensitive_keyword(name)
)
if not organization:
return header_scope
return header_scope | {("openai-organization", str(organization))}
@staticmethod
def _header_items(litellm_params: Mapping[str, object]) -> Iterator[tuple[str, object]]:
@ -313,8 +309,6 @@ class EncryptedContentAffinityCheck(CustomLogger):
@staticmethod
def _default_api_key(route: _ProviderRoute) -> str | None:
if route.provider not in _PROVIDER_NAMES:
return None
model_info: Final = ProviderConfigManager.get_provider_model_info(
model=route.model, provider=LlmProviders(route.provider)
)

View file

@ -2730,14 +2730,12 @@ def _deployment_of(ctx: RoutingContext, model_id: str) -> dict[str, object]:
SUMMARY_ONLY_REASONING: Final = {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]}
UNMARKED_REASONING: Final = {**SUMMARY_ONLY_REASONING, "id": "rs_test", "encrypted_content": "gAAAAA-blob"}
OPENAI_SHARED_KEY: Final = {"model": "openai/gpt-5.5", "api_key": "shared-key"}
MANTLE_SHARED_KEY: Final = {"model": "bedrock_mantle/openai.gpt-6-astra", "api_key": "shared-key"}
@pytest.fixture
def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "api_base", None)
for name in ("AZURE_API_BASE", "BEDROCK_MANTLE_API_BASE"):
monkeypatch.delenv(name, raising=False)
monkeypatch.delenv("AZURE_API_BASE", raising=False)
@pytest.mark.usefixtures("no_process_wide_api_base")
@ -2754,22 +2752,16 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None:
),
pytest.param(OPENAI_SHARED_KEY, {**OPENAI_SHARED_KEY, "api_key": "other-key"}, False, id="different key"),
pytest.param(
{**MANTLE_SHARED_KEY, "aws_region_name": "us-east-2"},
{**MANTLE_SHARED_KEY, "aws_region_name": "us-east-1"},
{"model": "bedrock_mantle/us-west-2/openai.gpt-6-astra", "api_key": "shared-key"},
{"model": "bedrock_mantle/us-west-2/openai.gpt-6-astra", "api_key": "shared-key"},
False,
id="same key in different Mantle regions",
id="same key on Bedrock Mantle, which serves several vendors",
),
pytest.param(
{**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"},
{**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-east-1/openai.gpt-6-astra"},
{"model": "openrouter/openai/gpt-6.1-sol", "api_key": "shared-key"},
{"model": "openrouter/anthropic/claude-haiku-5.5", "api_key": "shared-key"},
False,
id="same key in different Mantle regions named in the model",
),
pytest.param(
{**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"},
{**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"},
True,
id="same key in one Mantle region named in the model",
id="same key on an aggregator that serves several vendors",
),
pytest.param(
{"model": "anthropic/claude-opus-5-5", "api_key": "shared-key"},
@ -2819,11 +2811,17 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None:
False,
id="same key with an OpenAI project beside one without",
),
pytest.param(
{**OPENAI_SHARED_KEY, "extra_headers": {"Cookie": "session=account-a"}},
{**OPENAI_SHARED_KEY, "extra_headers": {"cookie": "session=account-b"}},
False,
id="same key signed in to different accounts by cookie",
),
pytest.param(
{**OPENAI_SHARED_KEY, "organization": "org-a"},
{**OPENAI_SHARED_KEY, "headers": {"OpenAI-Organization": "org-a"}},
{**OPENAI_SHARED_KEY, "organization": "org-b"},
True,
id="same OpenAI organization set two ways",
id="organization param the Responses request never sends",
),
pytest.param(
{**OPENAI_SHARED_KEY, "extra_headers": {"x-team": "a", "OpenAI-Organization": "org-a"}},