From b8ce24e51fb1c0b46e7e761a69c20a9a6847ceed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:53:51 -0700 Subject: [PATCH 01/10] fix(router): keep encrypted reasoning on a hop between same-key deployments on the provider's default endpoint --- .../encrypted_content_affinity_check.py | 87 +++++++---- ...allback_hop_foreign_encrypted_reasoning.py | 36 ++++- .../test_encrypted_content_affinity_check.py | 138 ++++++++++++++++++ 3 files changed, 230 insertions(+), 31 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 806c3db7601..20a1195fe3d 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -40,21 +40,24 @@ from collections.abc import Iterator, Mapping, Sequence from functools import cache from typing import TYPE_CHECKING, Final, Optional, cast -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter from litellm._logging import verbose_router_logger +from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_content_lists, encrypted_content_of_block, strip_encrypted_reasoning_from_messages, ) +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids from litellm.types.llms.openai import AllMessageValues from litellm.types.router import Deployment -from litellm.utils import get_order_filtered_deployments +from litellm.utils import ProviderConfigManager, get_order_filtered_deployments if TYPE_CHECKING: from litellm.router import Router @@ -209,40 +212,66 @@ class EncryptedContentAffinityCheck(CustomLogger): """ ``(api_base, api_key)`` identifies an upstream encryption boundary. The values are resolved from the deployment and its named credential - without modifying the deployment. + without modifying the deployment, and a deployment with no ``api_base`` + takes the endpoint its provider's Responses URL builder resolves for it. - Accepts any object exposing dict-style ``.get(key, default)``: plain - dicts (the common case in ``healthy_deployments``) as well as - ``LiteLLM_Params``-style Pydantic instances, which define a custom - ``.get()``. A stricter ``isinstance(dict)`` guard would silently drop - the latter from boundary matching and fall back to the full pool — - i.e. trigger the exact ``invalid_encrypted_content`` failure this - check exists to prevent. + Accepts plain dicts (the common case in ``healthy_deployments``) as + well as ``LiteLLM_Params`` Pydantic instances. A stricter + ``isinstance(dict)`` guard would silently drop the latter from + boundary matching and fall back to the full pool — i.e. trigger the + exact ``invalid_encrypted_content`` failure this check exists to prevent. """ - getter: Final = getattr(litellm_params, "get", None) - if not callable(getter): + params: Final = EncryptedContentAffinityCheck._params_mapping(litellm_params) + if params is None: return None - api_base: Final = getter("api_base") - api_key: Final = getter("api_key") - credential_name: Final = getter("litellm_credential_name") - credential_values: Final[Mapping[str, object] | None] = ( + credential_name: Final = params.get("litellm_credential_name") + credential_values: Final[Mapping[str, object]] = ( CredentialAccessor.get_credential_values(credential_name) if isinstance(credential_name, str) and credential_name - else None + else {} ) - effective_api_base: Final = ( - credential_values.get("api_base") - if credential_values is not None and "api_base" in credential_values - else api_base - ) - effective_api_key: Final = ( - credential_values.get("api_key") - if credential_values is not None and "api_key" in credential_values - else api_key - ) - if not effective_api_base or not effective_api_key: + effective_params: Final = {**params, **credential_values} + effective_api_key: Final = effective_params.get("api_key") + if not effective_api_key: + return None + api_base: Final = effective_params.get("api_base") or EncryptedContentAffinityCheck._default_responses_endpoint( + effective_params + ) + if not api_base: + return None + return (api_base, effective_api_key) + + @staticmethod + def _params_mapping(litellm_params: object) -> Mapping[str, object] | None: + if isinstance(litellm_params, BaseModel): + return litellm_params.model_dump(exclude_none=True) + if isinstance(litellm_params, Mapping): + return _REQUEST_KWARGS_ADAPTER.validate_python(litellm_params) + return None + + @staticmethod + def _default_responses_endpoint(litellm_params: Mapping[str, object]) -> str | None: + model: Final = litellm_params.get("model") + custom_llm_provider: Final = litellm_params.get("custom_llm_provider") + if not isinstance(model, str): + return None + try: + provider_model, provider = resolve_model_and_provider_for_metadata( + model, custom_llm_provider if isinstance(custom_llm_provider, str) else None + ) + except BadRequestError: + return None + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( + provider=provider, model=provider_model + ) + if responses_config is None: + return None + try: + return responses_config.get_complete_url( # pyright: ignore[reportUnknownMemberType] # the base types litellm_params as a bare dict + api_base=None, litellm_params=dict(litellm_params) + ) + except (ValueError, BaseLLMException, AuthenticationError): return None - return (effective_api_base, effective_api_key) def _find_deployments_on_same_encryption_boundary( self, diff --git a/tests/integration/routing/test_fallback_hop_foreign_encrypted_reasoning.py b/tests/integration/routing/test_fallback_hop_foreign_encrypted_reasoning.py index e6ec3444369..ef2a9e5b72f 100644 --- a/tests/integration/routing/test_fallback_hop_foreign_encrypted_reasoning.py +++ b/tests/integration/routing/test_fallback_hop_foreign_encrypted_reasoning.py @@ -303,7 +303,7 @@ def scenario_api_base(scenario: Scenario, scenario_id: str, response: StoredResp def deployment( scenario: Scenario, group: str, - api_base: str, + api_base: str | None, *, order: int | None, model: str = RESPONSES_MODEL, @@ -316,7 +316,7 @@ def deployment( "litellm_params": { "model": model, "api_key": PROVIDER_KEY, - "api_base": api_base, + **({} if api_base is None else {"api_base": api_base}), **({} if order is None else {"order": order}), **extra_params, }, @@ -863,6 +863,38 @@ def test_hop_inside_one_encryption_boundary_keeps_the_reasoning(gateway: Gateway assert [body.get("input") for body in bodies] == [history(mark), history(mark)], bodies +def test_hop_between_same_key_deployments_on_the_providers_default_endpoint_keeps_the_reasoning( + gateway: Gateway, tmp_path: Path +) -> None: + shared: Final = f"default-{uuid.uuid4().hex[:12]}" + with ( + scratch_database() as database, + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": database, "OPENAI_BASE_URL": f"{gateway.upstream_url}/{shared}"}, + remove_environment=("DIRECT_URL", "DATABASE_URL_READ_REPLICA", "OPENAI_API_BASE"), + ) as owned, + owned.gateway.scenario() as scenario, + ): + owned.gateway.post("/config/update", {"router_settings": {"num_retries": 0}}) + one: Final = scenario_api_base(scenario, f"{shared}-o1", failing()).rsplit("/", 1)[-1] + two: Final = scenario_api_base(scenario, f"{shared}-o2", healthy_json()).rsplit("/", 1)[-1] + order_one: Final = deployment( + scenario, shared, None, order=1, extra_params={"extra_headers": {"x-scripted-scenario": one}} + ) + order_two: Final = deployment( + scenario, shared, None, order=2, extra_params={"extra_headers": {"x-scripted-scenario": two}} + ) + mark: Final = marker() + answer: Final = responses_answer( + owned.gateway, HTTPX, Target(shared, frozenset({order_one, order_two})), history(mark) + ) + assert_served_by(answer, order_two) + bodies: Final = bodies_for(observed(owned.gateway), shared) + assert [body.get("input") for body in bodies] == [history(mark), history(mark)], bodies + + def assert_cross_group_hop_stripped(gateway: Gateway, source_scenario: str, target_scenario: str, mark: str) -> None: records: Final = observed(gateway) assert [body.get("input") for body in bodies_for(records, source_scenario)] == [history(mark)], records diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index a3745c7b23f..8ff379c8f73 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -2715,3 +2715,141 @@ async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_th assert [deployment["model_info"]["id"] for deployment in result] == ["region-a"] assert reasoning_input[0] == original_reasoning + + +def _default_endpoint_deployment(model_id: str, litellm_params: dict[str, object]) -> dict[str, object]: + return {"model_name": "gpt-5.5", "litellm_params": litellm_params, "model_info": {"id": model_id}} + + +def _deployment_of(router: litellm.Router, model_id: str) -> dict[str, object]: + return next(deployment for deployment in router.model_list if deployment["model_info"]["id"] == model_id) + + +SUMMARY_ONLY_REASONING: Final = {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]} +UNMARKED_REASONING: Final = {**SUMMARY_ONLY_REASONING, "id": "rs_test", "encrypted_content": "gAAAAA-blob"} +OPENAI_SHARED_KEY: Final = {"model": "openai/gpt-5.5", "api_key": "shared-key"} +MANTLE_SHARED_KEY: Final = {"model": "bedrock_mantle/openai.gpt-6-astra", "api_key": "shared-key"} + + +@pytest.fixture +def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "api_base", None) + for name in ("AZURE_API_BASE", "BEDROCK_MANTLE_API_BASE"): + monkeypatch.delenv(name, raising=False) + + +@pytest.mark.usefixtures("no_process_wide_api_base") +@pytest.mark.parametrize("marked", [True, False], ids=["marked origin", "unmarked origin"]) +@pytest.mark.parametrize( + "origin_params, target_params, target_can_decrypt", + [ + pytest.param(OPENAI_SHARED_KEY, OPENAI_SHARED_KEY, True, id="same key on the provider default endpoint"), + pytest.param( + {"model": "gpt-5.5", "custom_llm_provider": "openai", "api_key": "shared-key"}, + OPENAI_SHARED_KEY, + True, + id="provider named apart from the model", + ), + pytest.param(OPENAI_SHARED_KEY, {**OPENAI_SHARED_KEY, "api_key": "other-key"}, False, id="different key"), + pytest.param( + {**MANTLE_SHARED_KEY, "aws_region_name": "us-east-2"}, + {**MANTLE_SHARED_KEY, "aws_region_name": "us-east-1"}, + False, + id="same key in different Mantle regions", + ), + pytest.param( + {"model": "anthropic/claude-opus-5-5", "api_key": "shared-key"}, + {"model": "anthropic/claude-opus-5-5", "api_key": "shared-key"}, + False, + id="provider without a Responses URL builder", + ), + pytest.param( + {"model": "azure/gpt-5.5", "api_key": "shared-key"}, + {"model": "azure/gpt-5.5", "api_key": "shared-key"}, + False, + id="provider whose builder needs an api_base", + ), + pytest.param( + {**OPENAI_SHARED_KEY, "api_base": "https://openai-proxy.example.com/v1"}, + OPENAI_SHARED_KEY, + False, + id="explicit api_base beside the default endpoint", + ), + ], +) +def test_hop_strip_keeps_reasoning_between_same_key_deployments_on_the_default_endpoint( + origin_params: dict[str, object], target_params: dict[str, object], target_can_decrypt: bool, marked: bool +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", origin_params), + _default_endpoint_deployment("target", target_params), + ] + ) + reasoning: Final = _reasoning_input_minted_by("origin")[0] if marked else dict(UNMARKED_REASONING) + original_reasoning: Final = dict(reasoning) + request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + router, + request_input, + None, + (_deployment_of(router, "target"),), + unmarked_origin=None if marked else "origin", + ) + + assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) + + +def test_boundary_key_puts_pydantic_and_plain_params_on_one_default_endpoint_boundary(): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + from litellm.types.router import LiteLLM_Params + + pydantic_key: Final = EncryptedContentAffinityCheck._encryption_boundary_key(LiteLLM_Params(**OPENAI_SHARED_KEY)) + + assert pydantic_key is not None + assert pydantic_key == EncryptedContentAffinityCheck._encryption_boundary_key(dict(OPENAI_SHARED_KEY)) + + +def test_boundary_key_has_no_default_endpoint_for_a_model_no_provider_claims(): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + unmapped: Final = {"model": "unmapped-model-name", "api_key": "shared-key"} + + assert EncryptedContentAffinityCheck._encryption_boundary_key(unmapped) is None + + +@pytest.mark.asyncio +async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_the_origin_cannot_serve(): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", OPENAI_SHARED_KEY), + _default_endpoint_deployment("peer", OPENAI_SHARED_KEY), + _default_endpoint_deployment("other-org", {**OPENAI_SHARED_KEY, "api_key": "other-key"}), + ] + ) + healthy: Final = [_deployment_of(router, "peer"), _deployment_of(router, "other-org")] + reasoning_input: Final = _reasoning_input_minted_by("origin") + original_reasoning: Final = dict(reasoning_input[0]) + + result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( + model="gpt-5.5", + healthy_deployments=healthy, + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input}, + ) + + assert [deployment["model_info"]["id"] for deployment in result] == ["peer"] + assert reasoning_input[0] == original_reasoning From 243f6fd3c38ab2a238ff45dceb095c581b3aac54 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:47:27 -0700 Subject: [PATCH 02/10] test(router): type the default-endpoint boundary test helpers --- .../test_encrypted_content_affinity_check.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 8ff379c8f73..bd95a6fea25 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,6 +16,7 @@ The mechanism works without any cache and supports two encoding strategies: """ import time +from collections.abc import Mapping from typing import Final from unittest.mock import AsyncMock, patch @@ -2717,12 +2718,14 @@ async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_th assert reasoning_input[0] == original_reasoning -def _default_endpoint_deployment(model_id: str, litellm_params: dict[str, object]) -> dict[str, object]: - return {"model_name": "gpt-5.5", "litellm_params": litellm_params, "model_info": {"id": model_id}} +def _default_endpoint_deployment(model_id: str, litellm_params: Mapping[str, object]) -> dict[str, object]: + return {"model_name": "gpt-5.5", "litellm_params": dict(litellm_params), "model_info": {"id": model_id}} def _deployment_of(router: litellm.Router, model_id: str) -> dict[str, object]: - return next(deployment for deployment in router.model_list if deployment["model_info"]["id"] == model_id) + deployment: Final = router.get_deployment(model_id=model_id) + assert deployment is not None + return deployment.model_dump(exclude_none=True) SUMMARY_ONLY_REASONING: Final = {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]} @@ -2840,16 +2843,16 @@ async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_t _default_endpoint_deployment("other-org", {**OPENAI_SHARED_KEY, "api_key": "other-key"}), ] ) - healthy: Final = [_deployment_of(router, "peer"), _deployment_of(router, "other-org")] + peer: Final = _deployment_of(router, "peer") reasoning_input: Final = _reasoning_input_minted_by("origin") original_reasoning: Final = dict(reasoning_input[0]) - result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( + result: Final[list[dict[str, object]]] = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( model="gpt-5.5", - healthy_deployments=healthy, + healthy_deployments=[peer, _deployment_of(router, "other-org")], messages=None, request_kwargs={"litellm_metadata": {}, "input": reasoning_input}, ) - assert [deployment["model_info"]["id"] for deployment in result] == ["peer"] + assert result == [peer] assert reasoning_input[0] == original_reasoning From 63b22cb1420d6bfd6389e5ca4f8dfcbe1cb8e30a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:26:19 -0700 Subject: [PATCH 03/10] fix(guardrails): restore the google_text_moderation embeddings fix an earlier main merge dropped --- .../google_text_moderation.py | 10 ++- litellm/proxy/guardrails/_content_utils.py | 14 ++++ .../proxy/guardrails/test_content_utils.py | 25 ++++++ .../guardrails/test_guardrail_coverage.py | 79 ++++++++++++++++++- 4 files changed, 125 insertions(+), 3 deletions(-) diff --git a/enterprise/enterprise_hooks/google_text_moderation.py b/enterprise/enterprise_hooks/google_text_moderation.py index 7dbbc9ed6ce..ddf092a143e 100644 --- a/enterprise/enterprise_hooks/google_text_moderation.py +++ b/enterprise/enterprise_hooks/google_text_moderation.py @@ -15,7 +15,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.guardrails._content_utils import iter_message_text +from litellm.proxy.guardrails._content_utils import ( + is_non_conversational_request, + iter_message_text, +) from litellm.types.utils import CallTypesLiteral _THRESHOLD_VALUE: Final = TypeAdapter(object) @@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): async def async_moderation_hook( self, - data: dict, + data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, ): @@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): - Calls Google's Text Moderation API - Rejects request if it fails safety check """ + if is_non_conversational_request(call_type, data): + verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type) + return # Covers multimodal list content + Responses-API input. text = "".join(iter_message_text(data)) if text: diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 9bea22acd71..51934c99db6 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool: return call_type in NON_CONVERSATIONAL_CALL_TYPES +_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?") + + +def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool: + if is_non_conversational_call_type(call_type): + return True + endpoint: Final = data.get("endpoint") + return ( + call_type == "allm_passthrough_route" + and isinstance(endpoint, str) + and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None + ) + + TEXT_PART_TYPES: Final[frozenset[str]] = frozenset( {"text", "input_text", "output_text", "summary_text", "reasoning_text"} ) diff --git a/tests/unit/proxy/guardrails/test_content_utils.py b/tests/unit/proxy/guardrails/test_content_utils.py index 21d96a3ea97..c50646d233d 100644 --- a/tests/unit/proxy/guardrails/test_content_utils.py +++ b/tests/unit/proxy/guardrails/test_content_utils.py @@ -12,6 +12,7 @@ from litellm.proxy.guardrails._content_utils import ( build_inspection_messages, has_non_string_content, is_non_conversational_call_type, + is_non_conversational_request, is_string_batch_input, iter_message_text, iter_tool_json_text, @@ -819,6 +820,30 @@ def test_is_non_conversational_call_type_defaults_to_inspecting_unknown_call_typ assert is_non_conversational_call_type("some_future_call_type") is False +@pytest.mark.parametrize( + ("call_type", "endpoint", "expected"), + [ + ("aembedding", None, True), + ("allm_passthrough_route", "embeddings", True), + ("allm_passthrough_route", "/v1/embeddings/", True), + ("allm_passthrough_route", "chat/completions", False), + ("allm_passthrough_route", "model/amazon.titan-embed-text-v2:0/invoke", False), + ("allm_passthrough_route", "fine_tuned_embeddings_jobs", False), + ("allm_passthrough_route", "chat/completions#embeddings", False), + ("allm_passthrough_route", "chat/completions?x=embeddings", False), + ("allm_passthrough_route", "chat/completions;embeddings", False), + ("allm_passthrough_route", None, False), + ("acompletion", "embeddings", False), + ], +) +def test_is_non_conversational_request_skips_only_embeddings_requests( + call_type: str, endpoint: str | None, expected: bool +): + data: Final[dict[str, object]] = {"endpoint": endpoint} if endpoint is not None else {} + + assert is_non_conversational_request(call_type, data) is expected + + @pytest.mark.parametrize( "tool_input", ( diff --git a/tests/unit/proxy/guardrails/test_guardrail_coverage.py b/tests/unit/proxy/guardrails/test_guardrail_coverage.py index 1feae213eba..493eac430d0 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_coverage.py +++ b/tests/unit/proxy/guardrails/test_guardrail_coverage.py @@ -12,17 +12,19 @@ inspection by the relevant guardrail hook: not just ``choices[0]``. """ +from types import SimpleNamespace from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import HTTPException from httpx import Request, Response import litellm from litellm import DualCache from litellm.constants import DEFAULT_OPENAI_MODERATIONS_MODEL from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse @pytest.fixture @@ -905,3 +907,78 @@ async def test_google_text_moderation_inspects_multimodal_content(user_api_key): ) assert seen_documents == ["hello world"] + + +class _FinanceFlaggingLanguageClient: + def __init__(self) -> None: + self.moderated_texts: Final[list[str]] = [] # mutable-ok: records what reached Google + + def moderate_text(self, request: SimpleNamespace) -> SimpleNamespace: + self.moderated_texts.append(request.document.content) + return SimpleNamespace(moderation_categories=[SimpleNamespace(name="Finance", confidence=0.885)]) + + +def _google_text_moderation_flagging_finance(): + from enterprise.enterprise_hooks.google_text_moderation import ( + ENTERPRISE_GoogleTextModeration, + ) + + guard: Final = ENTERPRISE_GoogleTextModeration.__new__(ENTERPRISE_GoogleTextModeration) + guard.client = _FinanceFlaggingLanguageClient() + guard.language_document = lambda content, type_: SimpleNamespace(content=content) + guard.moderate_text_request = lambda document: SimpleNamespace(document=document) + guard.document_type = "PLAIN_TEXT" + guard.finance_confidence_threshold = 0.8 + return guard + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ("aembedding", {"model": "text-embedding-ada-002", "input": "stocks"}), + ("embedding", {"model": "text-embedding-ada-002", "input": ["stocks", "mortgage"]}), + ("allm_passthrough_route", {"model": "gigachat-embeddings", "endpoint": "embeddings", "input": ["stocks"]}), + ], +) +async def test_google_text_moderation_leaves_embedding_input_unmoderated( + user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object] +): + guard: Final = _google_text_moderation_flagging_finance() + + await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type) + + assert guard.client.moderated_texts == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ("acompletion", {"model": "gpt-4o", "messages": [{"role": "user", "content": "stocks"}]}), + ("anthropic_messages", {"model": "claude", "messages": [{"role": "user", "content": "stocks"}]}), + ("aresponses", {"model": "gpt-4o", "input": "stocks"}), + ( + "allm_passthrough_route", + {"model": "gigachat", "endpoint": "chat/completions", "messages": [{"role": "user", "content": "stocks"}]}, + ), + ( + "allm_passthrough_route", + { + "model": "gigachat", + "endpoint": "chat/completions#embeddings", + "messages": [{"role": "user", "content": "stocks"}], + }, + ), + ], +) +async def test_google_text_moderation_still_blocks_conversations( + user_api_key: UserAPIKeyAuth, call_type: CallTypesLiteral, data: dict[str, object] +): + guard: Final = _google_text_moderation_flagging_finance() + + with pytest.raises(HTTPException) as exc_info: + await guard.async_moderation_hook(data=data, user_api_key_dict=user_api_key, call_type=call_type) + + assert exc_info.value.status_code == 400 + assert guard.client.moderated_texts == ["stocks"] From 5e361c59904938d929a8c54791ae38619712f297 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 03:48:04 -0700 Subject: [PATCH 04/10] fix(router): match encryption boundaries on the key, account, and region the request really uses A deployment's named credential now fills only the values the deployment leaves out, through the same helper the request path uses, so the boundary names the key and base the request sends. A deployment with no api_key takes the provider's default key, the boundary carries the OpenAI organization and project a deployment sends, and a Bedrock Mantle region named in the model routes through the provider resolver before the Responses URL builder --- .../encrypted_content_affinity_check.py | 107 +++++++++--- .../test_encrypted_content_affinity_check.py | 165 +++++++++++++----- 2 files changed, 206 insertions(+), 66 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 6f9554576b8..379042a1c38 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,6 +37,7 @@ Safe to enable globally: """ from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import cache from typing import TYPE_CHECKING, Final, Optional, cast @@ -45,8 +46,7 @@ from pydantic import BaseModel, TypeAdapter from litellm._logging import verbose_router_logger from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_content_lists, encrypted_content_of_block, @@ -57,13 +57,23 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param from litellm.types.llms.openai import AllMessageValues -from litellm.types.router import Deployment -from litellm.utils import ProviderConfigManager, get_order_filtered_deployments +from litellm.types.router import Deployment, GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, get_order_filtered_deployments, with_named_credential if TYPE_CHECKING: from litellm.router_api import RoutingContext _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project"}) +_PROVIDER_NAMES: Final = frozenset(provider.value for provider in LlmProviders) + + +@dataclass(frozen=True, slots=True) +class _ProviderRoute: + model: str + provider: str + api_base: str | None class EncryptedContentAffinityCheck(CustomLogger): @@ -212,12 +222,13 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _encryption_boundary_key( litellm_params: object, - ) -> tuple[object, object] | None: + ) -> tuple[object, object, frozenset[tuple[str, object]]] | None: """ - ``(api_base, api_key)`` identifies an upstream encryption boundary. - The values are resolved from the deployment and its named credential - without modifying the deployment, and a deployment with no ``api_base`` - takes the endpoint its provider's Responses URL builder resolves for it. + ``(api_base, api_key, account scope)`` identifies an upstream encryption boundary. + The values are resolved the way the request resolves them, the deployment's own values + winning over its named credential's, without modifying the deployment. A deployment with + no ``api_key`` takes its provider's default key, and one with no ``api_base`` takes the + endpoint its provider's Responses URL builder resolves for it. Accepts plain dicts (the common case in ``healthy_deployments``) as well as ``LiteLLM_Params`` Pydantic instances. A stricter @@ -228,22 +239,24 @@ class EncryptedContentAffinityCheck(CustomLogger): params: Final = EncryptedContentAffinityCheck._params_mapping(litellm_params) if params is None: return None - credential_name: Final = params.get("litellm_credential_name") - credential_values: Final[Mapping[str, object]] = ( - CredentialAccessor.get_credential_values(credential_name) - if isinstance(credential_name, str) and credential_name - else {} - ) - effective_params: Final = {**params, **credential_values} - effective_api_key: Final = effective_params.get("api_key") - if not effective_api_key: + effective_params: Final = with_named_credential(params) + account_scope: Final = EncryptedContentAffinityCheck._account_scope(effective_params) + explicit_api_key: Final = effective_params.get("api_key") + explicit_api_base: Final = effective_params.get("api_base") + if explicit_api_key and explicit_api_base: + return (explicit_api_base, explicit_api_key, account_scope) + route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) + if route is None: return None - api_base: Final = effective_params.get("api_base") or EncryptedContentAffinityCheck._default_responses_endpoint( - effective_params + api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route) + if not api_key: + return None + api_base: Final = explicit_api_base or EncryptedContentAffinityCheck._default_responses_endpoint( + route, effective_params ) if not api_base: return None - return (api_base, effective_api_key) + return (api_base, api_key, account_scope) @staticmethod def _params_mapping(litellm_params: object) -> Mapping[str, object] | None: @@ -254,25 +267,65 @@ class EncryptedContentAffinityCheck(CustomLogger): return None @staticmethod - def _default_responses_endpoint(litellm_params: Mapping[str, object]) -> str | None: + def _account_scope(litellm_params: Mapping[str, object]) -> frozenset[tuple[str, object]]: + organization: Final = litellm_params.get("organization") + header_scope: Final = frozenset( + (name.lower(), value) + for name, value in EncryptedContentAffinityCheck._header_items(litellm_params) + if name.lower() in _ACCOUNT_SCOPE_HEADERS + ) + if not organization: + return header_scope + return header_scope | {("openai-organization", organization)} + + @staticmethod + def _header_items(litellm_params: Mapping[str, object]) -> Iterator[tuple[str, object]]: + for headers in (litellm_params.get("headers"), litellm_params.get("extra_headers")): + if isinstance(headers, Mapping): + yield from _REQUEST_KWARGS_ADAPTER.validate_python(headers).items() + + @staticmethod + def _provider_route(litellm_params: Mapping[str, object]) -> _ProviderRoute | None: model: Final = litellm_params.get("model") - custom_llm_provider: Final = litellm_params.get("custom_llm_provider") if not isinstance(model, str): return None + custom_llm_provider: Final = litellm_params.get("custom_llm_provider") + if declared_authenticating_provider( + model, custom_llm_provider if isinstance(custom_llm_provider, str) else None + ): + return None try: - provider_model, provider = resolve_model_and_provider_for_metadata( - model, custom_llm_provider if isinstance(custom_llm_provider, str) else None + provider_model, provider, _, provider_api_base = get_llm_provider( + model=model, litellm_params=GenericLiteLLMParams.model_validate(dict(litellm_params)) ) except BadRequestError: return None + return _ProviderRoute(provider_model, provider, provider_api_base) + + @staticmethod + def _default_api_key(route: _ProviderRoute) -> str | None: + if route.provider not in _PROVIDER_NAMES: + return None + model_info: Final = ProviderConfigManager.get_provider_model_info( + model=route.model, provider=LlmProviders(route.provider) + ) + if model_info is None: + return None + try: + return model_info.get_api_key() + except NotImplementedError: + return None + + @staticmethod + def _default_responses_endpoint(route: _ProviderRoute, litellm_params: Mapping[str, object]) -> str | None: responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( - provider=provider, model=provider_model + provider=route.provider, model=route.model ) if responses_config is None: return None try: return responses_config.get_complete_url( # pyright: ignore[reportUnknownMemberType] # the base types litellm_params as a bare dict - api_base=None, litellm_params=dict(litellm_params) + api_base=route.api_base, litellm_params=dict(litellm_params) ) except (ValueError, BaseLLMException, AuthenticationError): return None diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index b5a48f22329..785b38d2715 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1107,6 +1107,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance(): == ( "https://mateo-resource.openai.azure.com", "fake-azure-resource-key-a", + frozenset(), ) ) @@ -1132,14 +1133,29 @@ def test_boundary_key_resolves_missing_values_from_named_credential(): ): boundary = EncryptedContentAffinityCheck._encryption_boundary_key({"litellm_credential_name": "account-a"}) - assert boundary == ("https://account-a.example.com", "credential-key-a") + assert boundary == ("https://account-a.example.com", "credential-key-a", frozenset()) -def test_boundary_key_matches_named_credential_precedence(): +@pytest.mark.parametrize( + "deployment_values", + [ + pytest.param({"api_base": "https://deployment.example.com", "api_key": "deployment-key"}, id="set values"), + pytest.param({"api_base": "", "api_key": ""}, id="empty values"), + ], +) +@pytest.mark.usefixtures("no_process_wide_api_base") +def test_boundary_key_resolves_named_credential_values_the_way_the_request_does( + monkeypatch: pytest.MonkeyPatch, deployment_values: dict[str, str] +): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( EncryptedContentAffinityCheck, ) + from litellm.utils import with_named_credential + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + deployment: Final = {"model": "openai/gpt-5.5", **deployment_values, "litellm_credential_name": "account-a"} with patch.object( litellm, "credential_list", @@ -1154,46 +1170,14 @@ def test_boundary_key_matches_named_credential_precedence(): ) ], ): - boundary = EncryptedContentAffinityCheck._encryption_boundary_key( - { - "api_base": "https://deployment.example.com", - "api_key": "deployment-key", - "litellm_credential_name": "account-a", - } - ) + boundary = EncryptedContentAffinityCheck._encryption_boundary_key(deployment) + sent_params = with_named_credential(deployment) - assert boundary == ("https://credential.example.com", "credential-key-a") - - -def test_boundary_key_resolves_credential_when_explicit_values_are_empty(): - from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( - EncryptedContentAffinityCheck, + assert boundary is not None + assert boundary == EncryptedContentAffinityCheck._encryption_boundary_key( + {key: value for key, value in sent_params.items() if key != "litellm_credential_name"} ) - with patch.object( - litellm, - "credential_list", - [ - CredentialItem( - credential_name="account-a", - credential_values={ - "api_base": "https://credential.example.com", - "api_key": "credential-key-a", - }, - credential_info={}, - ) - ], - ): - boundary = EncryptedContentAffinityCheck._encryption_boundary_key( - { - "api_base": "", - "api_key": "", - "litellm_credential_name": "account-a", - } - ) - - assert boundary == ("https://credential.example.com", "credential-key-a") - def test_boundary_fallback_matches_deployments_with_same_named_credential_values(): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( @@ -2773,6 +2757,18 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: False, id="same key in different Mantle regions", ), + pytest.param( + {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"}, + {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-east-1/openai.gpt-6-astra"}, + False, + id="same key in different Mantle regions named in the model", + ), + pytest.param( + {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"}, + {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"}, + True, + id="same key in one Mantle region named in the model", + ), pytest.param( {"model": "anthropic/claude-opus-5-5", "api_key": "shared-key"}, {"model": "anthropic/claude-opus-5-5", "api_key": "shared-key"}, @@ -2785,12 +2781,54 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: False, id="provider whose builder needs an api_base", ), + pytest.param( + {"model": "github_copilot/gpt-5.5", "api_key": "shared-key"}, + {"model": "github_copilot/gpt-5.5", "api_key": "shared-key"}, + False, + id="same key on a provider that authenticates with its own token", + ), + pytest.param( + {"model": "groq/llama-4-scout"}, + {"model": "groq/llama-4-scout"}, + False, + id="provider with no default key", + ), + pytest.param( + {"model": "vertex_ai/gemini-3-pro"}, + {"model": "vertex_ai/gemini-3-pro"}, + False, + id="provider whose default key is a credentials file", + ), pytest.param( {**OPENAI_SHARED_KEY, "api_base": "https://openai-proxy.example.com/v1"}, OPENAI_SHARED_KEY, False, id="explicit api_base beside the default endpoint", ), + pytest.param( + {**OPENAI_SHARED_KEY, "extra_headers": {"OpenAI-Organization": "org-a"}}, + {**OPENAI_SHARED_KEY, "extra_headers": {"openai-organization": "org-b"}}, + False, + id="same key sent to different OpenAI organizations", + ), + pytest.param( + {**OPENAI_SHARED_KEY, "extra_headers": {"OpenAI-Project": "proj-a"}}, + OPENAI_SHARED_KEY, + False, + id="same key with an OpenAI project beside one without", + ), + pytest.param( + {**OPENAI_SHARED_KEY, "organization": "org-a"}, + {**OPENAI_SHARED_KEY, "headers": {"OpenAI-Organization": "org-a"}}, + True, + id="same OpenAI organization set two ways", + ), + pytest.param( + {**OPENAI_SHARED_KEY, "extra_headers": {"x-team": "a", "OpenAI-Organization": "org-a"}}, + {**OPENAI_SHARED_KEY, "extra_headers": {"x-team": "b", "OpenAI-Organization": "org-a"}}, + True, + id="same OpenAI organization beside unrelated headers", + ), ], ) def test_hop_strip_keeps_reasoning_between_same_key_deployments_on_the_default_endpoint( @@ -2822,6 +2860,44 @@ def test_hop_strip_keeps_reasoning_between_same_key_deployments_on_the_default_e assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) +@pytest.mark.usefixtures("no_process_wide_api_base") +@pytest.mark.parametrize( + "target_api_key, target_can_decrypt", + [ + pytest.param(None, True, id="both on the environment key"), + pytest.param("env-key", True, id="explicit key equal to the environment key"), + pytest.param("other-key", False, id="explicit key other than the environment key"), + ], +) +def test_hop_strip_resolves_a_missing_key_from_the_environment( + monkeypatch: pytest.MonkeyPatch, target_api_key: str | None, target_can_decrypt: bool +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + target_params: Final = {"model": "openai/gpt-5.5", **({"api_key": target_api_key} if target_api_key else {})} + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", {"model": "openai/gpt-5.5"}), + _default_endpoint_deployment("target", target_params), + ] + ) + ctx: Final = RoutingContext(router) + reasoning: Final = _reasoning_input_minted_by("origin")[0] + original_reasoning: Final = dict(reasoning) + request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + ctx, request_input, None, (_deployment_of(ctx, "target"),), unmarked_origin=None + ) + + assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) + + def test_boundary_key_puts_pydantic_and_plain_params_on_one_default_endpoint_boundary(): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( EncryptedContentAffinityCheck, @@ -2844,6 +2920,17 @@ def test_boundary_key_has_no_default_endpoint_for_a_model_no_provider_claims(): assert EncryptedContentAffinityCheck._encryption_boundary_key(unmapped) is None +def test_boundary_key_has_no_default_key_for_a_custom_provider(monkeypatch: pytest.MonkeyPatch): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "qa-custom"]) + monkeypatch.setattr(litellm, "_custom_providers", ["qa-custom"]) + + assert EncryptedContentAffinityCheck._encryption_boundary_key({"model": "qa-custom/some-model"}) is None + + @pytest.mark.asyncio async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_the_origin_cannot_serve(): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( From 9c3deef179ae0e157a92a266b213186502a1ad80 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:22:04 -0700 Subject: [PATCH 05/10] fix(router): keep header-signed accounts apart on a shared boundary and skip boundary work no reasoning needs Headers the request logger masks as secrets (Authorization, api-key, x-api-key) now join the boundary's account scope, with extra_headers overriding headers the way the request merges them, so deployments that sign in through a header on one base never share encrypted reasoning. Target boundaries are built only when a reasoning item from another deployment needs the comparison, and the Responses input strip asks only about encrypted reasoning items, so a pinned turn or fallback hop with nothing to compare makes no secret-manager reads. --- litellm/responses/utils.py | 15 +- .../encrypted_content_affinity_check.py | 40 +++-- .../test_encrypted_content_affinity_check.py | 141 +++++++++++++++++- 3 files changed, 175 insertions(+), 21 deletions(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 1a97c1cb48c..43ed9a3b96f 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -678,20 +678,25 @@ class ResponsesAPIRequestUtils: return items: Final = cast(list[object], request_input) # cast-ok: untyped client json stripped: Final = tuple( - ResponsesAPIRequestUtils._without_encrypted_reasoning(item) - if should_strip is None or (isinstance(item, Mapping) and should_strip(cast(Mapping[str, object], item))) + ResponsesAPIRequestUtils._without_encrypted_reasoning(reasoning) + if (reasoning := ResponsesAPIRequestUtils._encrypted_reasoning_item(item)) is not None + and (should_strip is None or should_strip(reasoning)) else item for item in items ) items[:] = (item for item in stripped if item is not None) @staticmethod - def _without_encrypted_reasoning(item: object) -> object | None: + def _encrypted_reasoning_item(item: object) -> Mapping[str, object] | None: if not isinstance(item, dict): - return item + return None reasoning: Final = cast(Mapping[str, object], item) # cast-ok: untyped client json if reasoning.get("type") != "reasoning" or not reasoning.get("encrypted_content"): - return reasoning + return None + return reasoning + + @staticmethod + def _without_encrypted_reasoning(reasoning: Mapping[str, object]) -> object | None: readable: Final = any( ResponsesAPIRequestUtils._has_readable_text(reasoning.get(key)) for key in ("summary", "content") ) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 379042a1c38..292f1fadd52 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -47,6 +47,7 @@ from litellm._logging import verbose_router_logger from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider +from litellm.litellm_core_utils.litellm_logging import has_sensitive_keyword from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_content_lists, encrypted_content_of_block, @@ -270,19 +271,27 @@ class EncryptedContentAffinityCheck(CustomLogger): def _account_scope(litellm_params: Mapping[str, object]) -> frozenset[tuple[str, object]]: organization: Final = litellm_params.get("organization") header_scope: Final = frozenset( - (name.lower(), value) + (name.lower(), str(value)) for name, value in EncryptedContentAffinityCheck._header_items(litellm_params) - if name.lower() in _ACCOUNT_SCOPE_HEADERS + if name.lower() in _ACCOUNT_SCOPE_HEADERS or has_sensitive_keyword(name) ) if not organization: return header_scope - return header_scope | {("openai-organization", organization)} + return header_scope | {("openai-organization", str(organization))} @staticmethod def _header_items(litellm_params: Mapping[str, object]) -> Iterator[tuple[str, object]]: - for headers in (litellm_params.get("headers"), litellm_params.get("extra_headers")): - if isinstance(headers, Mapping): - yield from _REQUEST_KWARGS_ADAPTER.validate_python(headers).items() + headers: Final = EncryptedContentAffinityCheck._header_mapping(litellm_params.get("headers")) + extra_headers: Final = EncryptedContentAffinityCheck._header_mapping(litellm_params.get("extra_headers")) + overridden: Final = frozenset(name.lower() for name in extra_headers) + yield from ((name, value) for name, value in headers.items() if name.lower() not in overridden) + yield from extra_headers.items() + + @staticmethod + def _header_mapping(headers: object) -> Mapping[str, object]: + if isinstance(headers, Mapping): + return _REQUEST_KWARGS_ADAPTER.validate_python(headers) + return {} @staticmethod def _provider_route(litellm_params: Mapping[str, object]) -> _ProviderRoute | None: @@ -337,7 +346,7 @@ class EncryptedContentAffinityCheck(CustomLogger): ) -> tuple[list[dict], Deployment | None]: """ Deployments in ``healthy_deployments`` sharing the originating - deployment's ``(api_base, api_key)``, alongside the originating + deployment's encryption boundary, alongside the originating deployment object (or ``None`` if it was removed / router unavailable). """ if self.ctx is None: @@ -375,12 +384,15 @@ class EncryptedContentAffinityCheck(CustomLogger): for target in target_deployments if isinstance((model_info := target.get("model_info")), Mapping) and model_info.get("id") is not None ) - target_boundaries: Final = frozenset( - boundary - for target in target_deployments - if (boundary := EncryptedContentAffinityCheck._encryption_boundary_key(target.get("litellm_params"))) - is not None - ) + + @cache + def target_boundaries() -> frozenset[tuple[object, object, frozenset[tuple[str, object]]]]: + return frozenset( + boundary + for target in target_deployments + if (boundary := EncryptedContentAffinityCheck._encryption_boundary_key(target.get("litellm_params"))) + is not None + ) @cache def target_can_decrypt(marked_origin: str | None) -> bool: @@ -399,7 +411,7 @@ class EncryptedContentAffinityCheck(CustomLogger): if origin is not None else None ) - return origin_boundary is not None and origin_boundary in target_boundaries + return origin_boundary is not None and origin_boundary in target_boundaries() def should_strip_input_item(item: Mapping[str, object]) -> bool: return not target_can_decrypt(EncryptedContentAffinityCheck._model_id_of_input_item(item)) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 785b38d2715..a746ce0a98b 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -23,10 +23,12 @@ from unittest.mock import AsyncMock, patch import pytest import litellm +from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.models.credentials import CredentialItem from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_api import RoutingContext from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem @pytest.fixture(autouse=True) @@ -1170,8 +1172,8 @@ def test_boundary_key_resolves_named_credential_values_the_way_the_request_does( ) ], ): - boundary = EncryptedContentAffinityCheck._encryption_boundary_key(deployment) - sent_params = with_named_credential(deployment) + boundary: Final = EncryptedContentAffinityCheck._encryption_boundary_key(deployment) + sent_params: Final = with_named_credential(deployment) assert boundary is not None assert boundary == EncryptedContentAffinityCheck._encryption_boundary_key( @@ -2829,6 +2831,16 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: True, id="same OpenAI organization beside unrelated headers", ), + pytest.param( + { + **OPENAI_SHARED_KEY, + "headers": {"OpenAI-Organization": "org-a"}, + "extra_headers": {"openai-organization": "org-b"}, + }, + {**OPENAI_SHARED_KEY, "extra_headers": {"OpenAI-Organization": "org-b"}}, + True, + id="extra headers override the organization the headers set", + ), ], ) def test_hop_strip_keeps_reasoning_between_same_key_deployments_on_the_default_endpoint( @@ -2898,6 +2910,131 @@ def test_hop_strip_resolves_a_missing_key_from_the_environment( assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) +def test_hop_strip_leaves_input_items_that_are_not_objects_as_sent(): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + request_input: Final = ["and sunsets?", *_reasoning_input_minted_by("origin")] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + None, request_input, None, (), unmarked_origin=None + ) + + assert request_input == ["and sunsets?", SUMMARY_ONLY_REASONING, {"role": "user", "content": "and sunsets?"}] + + +GATEWAY_BASE: Final = {"model": "openai/gpt-5.5", "api_base": "https://gateway.example.com/v1"} + + +@pytest.mark.parametrize( + "origin_params, target_params, target_can_decrypt", + [ + pytest.param( + {**GATEWAY_BASE, "extra_headers": {"Authorization": "Bearer sk-account-a"}}, + {**GATEWAY_BASE, "extra_headers": {"Authorization": "Bearer sk-account-b"}}, + False, + id="environment key replaced by different Authorization headers", + ), + pytest.param( + {**GATEWAY_BASE, "api_key": "shared-key", "headers": {"api-key": "account-a"}}, + {**GATEWAY_BASE, "api_key": "shared-key", "headers": {"api-key": "account-b"}}, + False, + id="explicit key beside different api-key headers", + ), + pytest.param( + {**GATEWAY_BASE, "extra_headers": {"Authorization": "Bearer sk-account-a"}}, + {**GATEWAY_BASE, "extra_headers": {"authorization": "Bearer sk-account-a"}}, + True, + id="same Authorization header on both", + ), + ], +) +def test_hop_strip_keeps_accounts_that_sign_in_through_headers_apart( + monkeypatch: pytest.MonkeyPatch, + origin_params: dict[str, object], + target_params: dict[str, object], + target_can_decrypt: bool, +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", origin_params), + _default_endpoint_deployment("target", target_params), + ] + ) + ctx: Final = RoutingContext(router) + reasoning: Final = _reasoning_input_minted_by("origin")[0] + original_reasoning: Final = dict(reasoning) + request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + ctx, request_input, None, (_deployment_of(ctx, "target"),), unmarked_origin=None + ) + + assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) + + +class _SecretReadCounter(CustomSecretManager): + def __init__(self) -> None: + super().__init__(secret_manager_name="read-counter") + self.reads: int = 0 + + async def async_read_secret( + self, secret_name: str, optional_params: Mapping[str, object] | None = None, timeout: object = None + ) -> str | None: + self.reads += 1 + return None + + def sync_read_secret( + self, secret_name: str, optional_params: Mapping[str, object] | None = None, timeout: object = None + ) -> str | None: + self.reads += 1 + return None + + +@pytest.mark.parametrize( + "reasoning_minted_by", + [ + pytest.param(None, id="no reasoning in the input"), + pytest.param("target", id="reasoning the hop's target minted"), + ], +) +def test_hop_strip_reads_no_secrets_when_no_reasoning_needs_a_boundary( + monkeypatch: pytest.MonkeyPatch, reasoning_minted_by: str | None +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", OPENAI_SHARED_KEY), + _default_endpoint_deployment("target", OPENAI_SHARED_KEY), + ] + ) + ctx: Final = RoutingContext(router) + secret_reads: Final = _SecretReadCounter() + monkeypatch.setattr(litellm, "secret_manager_client", secret_reads) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + request_input: Final = ( + _reasoning_input_minted_by(reasoning_minted_by) + if reasoning_minted_by + else [{"role": "user", "content": "and sunsets?"}] + ) + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + ctx, request_input, None, (_deployment_of(ctx, "target"),), unmarked_origin="origin" + ) + + assert secret_reads.reads == 0 + + def test_boundary_key_puts_pydantic_and_plain_params_on_one_default_endpoint_boundary(): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( EncryptedContentAffinityCheck, From f9e5a09e7bcc247fb0695502c5206e4f85f3410b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:14:57 -0700 Subject: [PATCH 06/10] fix(router): resolve default boundaries only for openai and drop the unsent organization param Aggregators and Bedrock Mantle serve several vendors on one key, so a missing key or base there no longer yields a shared boundary. The Responses request never sends the organization param, so it no longer splits the boundary, while a Cookie header now does. --- .../encrypted_content_affinity_check.py | 20 ++++-------- .../test_encrypted_content_affinity_check.py | 32 +++++++++---------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 292f1fadd52..b17f34e3cad 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -66,8 +66,8 @@ if TYPE_CHECKING: from litellm.router_api import RoutingContext _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project"}) -_PROVIDER_NAMES: Final = frozenset(provider.value for provider in LlmProviders) +_ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) +_SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) @dataclass(frozen=True, slots=True) @@ -227,9 +227,9 @@ class EncryptedContentAffinityCheck(CustomLogger): """ ``(api_base, api_key, account scope)`` identifies an upstream encryption boundary. The values are resolved the way the request resolves them, the deployment's own values - winning over its named credential's, without modifying the deployment. A deployment with - no ``api_key`` takes its provider's default key, and one with no ``api_base`` takes the - endpoint its provider's Responses URL builder resolves for it. + winning over its named credential's, without modifying the deployment. On a provider that + serves one vendor's models, a deployment with no ``api_key`` takes the provider's default + key, and one with no ``api_base`` takes the endpoint its Responses URL builder resolves. Accepts plain dicts (the common case in ``healthy_deployments``) as well as ``LiteLLM_Params`` Pydantic instances. A stricter @@ -247,7 +247,7 @@ class EncryptedContentAffinityCheck(CustomLogger): if explicit_api_key and explicit_api_base: return (explicit_api_base, explicit_api_key, account_scope) route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) - if route is None: + if route is None or route.provider not in _SINGLE_VENDOR_PROVIDERS: return None api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route) if not api_key: @@ -269,15 +269,11 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _account_scope(litellm_params: Mapping[str, object]) -> frozenset[tuple[str, object]]: - organization: Final = litellm_params.get("organization") - header_scope: Final = frozenset( + return frozenset( (name.lower(), str(value)) for name, value in EncryptedContentAffinityCheck._header_items(litellm_params) if name.lower() in _ACCOUNT_SCOPE_HEADERS or has_sensitive_keyword(name) ) - if not organization: - return header_scope - return header_scope | {("openai-organization", str(organization))} @staticmethod def _header_items(litellm_params: Mapping[str, object]) -> Iterator[tuple[str, object]]: @@ -313,8 +309,6 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _default_api_key(route: _ProviderRoute) -> str | None: - if route.provider not in _PROVIDER_NAMES: - return None model_info: Final = ProviderConfigManager.get_provider_model_info( model=route.model, provider=LlmProviders(route.provider) ) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index a746ce0a98b..fa33de809f8 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -2730,14 +2730,12 @@ def _deployment_of(ctx: RoutingContext, model_id: str) -> dict[str, object]: SUMMARY_ONLY_REASONING: Final = {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]} UNMARKED_REASONING: Final = {**SUMMARY_ONLY_REASONING, "id": "rs_test", "encrypted_content": "gAAAAA-blob"} OPENAI_SHARED_KEY: Final = {"model": "openai/gpt-5.5", "api_key": "shared-key"} -MANTLE_SHARED_KEY: Final = {"model": "bedrock_mantle/openai.gpt-6-astra", "api_key": "shared-key"} @pytest.fixture def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "api_base", None) - for name in ("AZURE_API_BASE", "BEDROCK_MANTLE_API_BASE"): - monkeypatch.delenv(name, raising=False) + monkeypatch.delenv("AZURE_API_BASE", raising=False) @pytest.mark.usefixtures("no_process_wide_api_base") @@ -2754,22 +2752,16 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: ), pytest.param(OPENAI_SHARED_KEY, {**OPENAI_SHARED_KEY, "api_key": "other-key"}, False, id="different key"), pytest.param( - {**MANTLE_SHARED_KEY, "aws_region_name": "us-east-2"}, - {**MANTLE_SHARED_KEY, "aws_region_name": "us-east-1"}, + {"model": "bedrock_mantle/us-west-2/openai.gpt-6-astra", "api_key": "shared-key"}, + {"model": "bedrock_mantle/us-west-2/openai.gpt-6-astra", "api_key": "shared-key"}, False, - id="same key in different Mantle regions", + id="same key on Bedrock Mantle, which serves several vendors", ), pytest.param( - {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"}, - {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-east-1/openai.gpt-6-astra"}, + {"model": "openrouter/openai/gpt-6.1-sol", "api_key": "shared-key"}, + {"model": "openrouter/anthropic/claude-haiku-5.5", "api_key": "shared-key"}, False, - id="same key in different Mantle regions named in the model", - ), - pytest.param( - {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"}, - {**MANTLE_SHARED_KEY, "model": "bedrock_mantle/us-west-2/openai.gpt-6-astra"}, - True, - id="same key in one Mantle region named in the model", + id="same key on an aggregator that serves several vendors", ), pytest.param( {"model": "anthropic/claude-opus-5-5", "api_key": "shared-key"}, @@ -2819,11 +2811,17 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: False, id="same key with an OpenAI project beside one without", ), + pytest.param( + {**OPENAI_SHARED_KEY, "extra_headers": {"Cookie": "session=account-a"}}, + {**OPENAI_SHARED_KEY, "extra_headers": {"cookie": "session=account-b"}}, + False, + id="same key signed in to different accounts by cookie", + ), pytest.param( {**OPENAI_SHARED_KEY, "organization": "org-a"}, - {**OPENAI_SHARED_KEY, "headers": {"OpenAI-Organization": "org-a"}}, + {**OPENAI_SHARED_KEY, "organization": "org-b"}, True, - id="same OpenAI organization set two ways", + id="organization param the Responses request never sends", ), pytest.param( {**OPENAI_SHARED_KEY, "extra_headers": {"x-team": "a", "OpenAI-Organization": "org-a"}}, From d96515e76a5bffb089387203ac5889671846fabb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:05:49 -0700 Subject: [PATCH 07/10] fix(router): match explicit pairs as on main and key default boundaries by model An explicit api_base plus api_key pair is the whole boundary again, so tag headers no longer split deployments that main kept together. The default OpenAI boundary now carries the model, since one OPENAI_BASE_URL can front several vendors, and a deployment that declares another provider gets no boundary before any secret lookup. --- .../get_llm_provider_logic.py | 6 ++- .../encrypted_content_affinity_check.py | 32 ++++++++------- .../test_encrypted_content_affinity_check.py | 40 +++++++++++++------ 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 38242291b71..94ba02bfbd7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -146,6 +146,10 @@ def handle_anthropic_text_model_custom_llm_provider( return model, custom_llm_provider +def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None: + return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None) + + def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None: """The authenticating provider this pair already names, or None. @@ -154,7 +158,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str and for a declared pair the resolver's answer is the declaration itself, so metadata callers adopt the declaration instead of resolving. """ - declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None) + declared: Final = declared_provider(model, custom_llm_provider) return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index b17f34e3cad..aab62dd5db5 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -39,14 +39,15 @@ Safe to enable globally: from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from functools import cache -from typing import TYPE_CHECKING, Final, Optional, cast +from typing import TYPE_CHECKING, Final, Optional, TypeAlias, cast from pydantic import BaseModel, TypeAdapter +import litellm from litellm._logging import verbose_router_logger from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span -from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider +from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider, get_llm_provider from litellm.litellm_core_utils.litellm_logging import has_sensitive_keyword from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_content_lists, @@ -68,6 +69,7 @@ if TYPE_CHECKING: _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) +_EncryptionBoundary: TypeAlias = tuple[object, object] | tuple[object, object, str, frozenset[tuple[str, object]]] @dataclass(frozen=True, slots=True) @@ -223,13 +225,15 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _encryption_boundary_key( litellm_params: object, - ) -> tuple[object, object, frozenset[tuple[str, object]]] | None: + ) -> _EncryptionBoundary | None: """ - ``(api_base, api_key, account scope)`` identifies an upstream encryption boundary. - The values are resolved the way the request resolves them, the deployment's own values - winning over its named credential's, without modifying the deployment. On a provider that - serves one vendor's models, a deployment with no ``api_key`` takes the provider's default - key, and one with no ``api_base`` takes the endpoint its Responses URL builder resolves. + The upstream encryption boundary a deployment's encrypted reasoning belongs to. The values + are resolved the way the request resolves them, the deployment's own values winning over its + named credential's, without modifying the deployment. An explicit ``(api_base, api_key)`` + pair is the whole boundary. On a provider that serves one vendor's models, a deployment with + no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint + its Responses URL builder resolves, and the boundary also carries the model and the headers + that pick an account. Accepts plain dicts (the common case in ``healthy_deployments``) as well as ``LiteLLM_Params`` Pydantic instances. A stricter @@ -241,11 +245,10 @@ class EncryptedContentAffinityCheck(CustomLogger): if params is None: return None effective_params: Final = with_named_credential(params) - account_scope: Final = EncryptedContentAffinityCheck._account_scope(effective_params) explicit_api_key: Final = effective_params.get("api_key") explicit_api_base: Final = effective_params.get("api_base") if explicit_api_key and explicit_api_base: - return (explicit_api_base, explicit_api_key, account_scope) + return (explicit_api_base, explicit_api_key) route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) if route is None or route.provider not in _SINGLE_VENDOR_PROVIDERS: return None @@ -257,7 +260,7 @@ class EncryptedContentAffinityCheck(CustomLogger): ) if not api_base: return None - return (api_base, api_key, account_scope) + return (api_base, api_key, route.model, EncryptedContentAffinityCheck._account_scope(effective_params)) @staticmethod def _params_mapping(litellm_params: object) -> Mapping[str, object] | None: @@ -295,9 +298,10 @@ class EncryptedContentAffinityCheck(CustomLogger): if not isinstance(model, str): return None custom_llm_provider: Final = litellm_params.get("custom_llm_provider") - if declared_authenticating_provider( + declared: Final = declared_provider( model, custom_llm_provider if isinstance(custom_llm_provider, str) else None - ): + ) + if declared in litellm.provider_list and declared not in _SINGLE_VENDOR_PROVIDERS: return None try: provider_model, provider, _, provider_api_base = get_llm_provider( @@ -380,7 +384,7 @@ class EncryptedContentAffinityCheck(CustomLogger): ) @cache - def target_boundaries() -> frozenset[tuple[object, object, frozenset[tuple[str, object]]]]: + def target_boundaries() -> frozenset[_EncryptionBoundary]: return frozenset( boundary for target in target_deployments diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index fa33de809f8..edd3b6d6e1d 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1109,7 +1109,6 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance(): == ( "https://mateo-resource.openai.azure.com", "fake-azure-resource-key-a", - frozenset(), ) ) @@ -1135,7 +1134,7 @@ def test_boundary_key_resolves_missing_values_from_named_credential(): ): boundary = EncryptedContentAffinityCheck._encryption_boundary_key({"litellm_credential_name": "account-a"}) - assert boundary == ("https://account-a.example.com", "credential-key-a", frozenset()) + assert boundary == ("https://account-a.example.com", "credential-key-a") @pytest.mark.parametrize( @@ -2751,6 +2750,12 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: id="provider named apart from the model", ), pytest.param(OPENAI_SHARED_KEY, {**OPENAI_SHARED_KEY, "api_key": "other-key"}, False, id="different key"), + pytest.param( + OPENAI_SHARED_KEY, + {**OPENAI_SHARED_KEY, "model": "openai/gpt-5.5-mini"}, + False, + id="same key on the default endpoint for another model", + ), pytest.param( {"model": "bedrock_mantle/us-west-2/openai.gpt-6-astra", "api_key": "shared-key"}, {"model": "bedrock_mantle/us-west-2/openai.gpt-6-astra", "api_key": "shared-key"}, @@ -2817,6 +2822,12 @@ def no_process_wide_api_base(monkeypatch: pytest.MonkeyPatch) -> None: False, id="same key signed in to different accounts by cookie", ), + pytest.param( + {**OPENAI_SHARED_KEY, "headers": {"api-key": "account-a"}}, + {**OPENAI_SHARED_KEY, "headers": {"api-key": "account-b"}}, + False, + id="explicit key beside different api-key headers", + ), pytest.param( {**OPENAI_SHARED_KEY, "organization": "org-a"}, {**OPENAI_SHARED_KEY, "organization": "org-b"}, @@ -2935,10 +2946,10 @@ GATEWAY_BASE: Final = {"model": "openai/gpt-5.5", "api_base": "https://gateway.e id="environment key replaced by different Authorization headers", ), pytest.param( - {**GATEWAY_BASE, "api_key": "shared-key", "headers": {"api-key": "account-a"}}, - {**GATEWAY_BASE, "api_key": "shared-key", "headers": {"api-key": "account-b"}}, - False, - id="explicit key beside different api-key headers", + {**GATEWAY_BASE, "api_key": "shared-key", "extra_headers": {"x-idempotency-key": "a"}}, + {**GATEWAY_BASE, "api_key": "shared-key", "extra_headers": {"x-idempotency-key": "b"}}, + True, + id="explicit pair beside different tag headers", ), pytest.param( {**GATEWAY_BASE, "extra_headers": {"Authorization": "Bearer sk-account-a"}}, @@ -2996,14 +3007,17 @@ class _SecretReadCounter(CustomSecretManager): @pytest.mark.parametrize( - "reasoning_minted_by", + "deployment_params, reasoning_minted_by", [ - pytest.param(None, id="no reasoning in the input"), - pytest.param("target", id="reasoning the hop's target minted"), + pytest.param(OPENAI_SHARED_KEY, None, id="no reasoning in the input"), + pytest.param(OPENAI_SHARED_KEY, "target", id="reasoning the hop's target minted"), + pytest.param( + {"model": "groq/llama-4-scout"}, "origin", id="reasoning from a provider with no default boundary" + ), ], ) -def test_hop_strip_reads_no_secrets_when_no_reasoning_needs_a_boundary( - monkeypatch: pytest.MonkeyPatch, reasoning_minted_by: str | None +def test_hop_strip_reads_no_secrets_when_no_default_boundary_needs_resolving( + monkeypatch: pytest.MonkeyPatch, deployment_params: dict[str, object], reasoning_minted_by: str | None ): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( EncryptedContentAffinityCheck, @@ -3011,8 +3025,8 @@ def test_hop_strip_reads_no_secrets_when_no_reasoning_needs_a_boundary( router: Final = litellm.Router( model_list=[ - _default_endpoint_deployment("origin", OPENAI_SHARED_KEY), - _default_endpoint_deployment("target", OPENAI_SHARED_KEY), + _default_endpoint_deployment("origin", deployment_params), + _default_endpoint_deployment("target", deployment_params), ] ) ctx: Final = RoutingContext(router) From c0a44d63198ee5d81eabcbb63f2c334b4ba6b092 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:11:30 -0700 Subject: [PATCH 08/10] fix(router): match an explicit key and the environment key on one api_base A deployment with an explicit api_base and api_key now shares a boundary with one on the same api_base that takes the same key from OPENAI_API_KEY, when the model and account headers also match. Two explicit pairs still match on base and key alone, as on main. --- .../encrypted_content_affinity_check.py | 57 ++++++++++++++----- .../test_encrypted_content_affinity_check.py | 53 ++++++++++++++--- 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index aab62dd5db5..c8739c2ff2a 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -39,7 +39,7 @@ Safe to enable globally: from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from functools import cache -from typing import TYPE_CHECKING, Final, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Final, Optional, cast from pydantic import BaseModel, TypeAdapter @@ -69,7 +69,6 @@ if TYPE_CHECKING: _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) -_EncryptionBoundary: TypeAlias = tuple[object, object] | tuple[object, object, str, frozenset[tuple[str, object]]] @dataclass(frozen=True, slots=True) @@ -79,6 +78,21 @@ class _ProviderRoute: api_base: str | None +@dataclass(frozen=True, slots=True) +class _EncryptionBoundary: + api_base: object + api_key: object + model: str | None + account_scope: frozenset[tuple[str, object]] + explicit_pair: bool + + def is_shared_with(self, other: "_EncryptionBoundary") -> bool: + if (self.api_base, self.api_key) != (other.api_base, other.api_key): + return False + both_explicit: Final = self.explicit_pair and other.explicit_pair + return both_explicit or (self.model, self.account_scope) == (other.model, other.account_scope) + + class EncryptedContentAffinityCheck(CustomLogger): """ Routes follow-up Responses API requests to the deployment that produced @@ -229,11 +243,11 @@ class EncryptedContentAffinityCheck(CustomLogger): """ The upstream encryption boundary a deployment's encrypted reasoning belongs to. The values are resolved the way the request resolves them, the deployment's own values winning over its - named credential's, without modifying the deployment. An explicit ``(api_base, api_key)`` - pair is the whole boundary. On a provider that serves one vendor's models, a deployment with - no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint - its Responses URL builder resolves, and the boundary also carries the model and the headers - that pick an account. + named credential's, without modifying the deployment. Two explicit ``(api_base, api_key)`` + pairs match on those values alone. On a provider that serves one vendor's models, a deployment + with no ``api_key`` takes the provider's default key, one with no ``api_base`` takes the endpoint + its Responses URL builder resolves, and a match involving either also needs the same model and + the same headers that pick an account. Accepts plain dicts (the common case in ``healthy_deployments``) as well as ``LiteLLM_Params`` Pydantic instances. A stricter @@ -247,10 +261,18 @@ class EncryptedContentAffinityCheck(CustomLogger): effective_params: Final = with_named_credential(params) explicit_api_key: Final = effective_params.get("api_key") explicit_api_base: Final = effective_params.get("api_base") + provider_route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) + route: Final = ( + provider_route + if provider_route is not None and provider_route.provider in _SINGLE_VENDOR_PROVIDERS + else None + ) + account_scope: Final = EncryptedContentAffinityCheck._account_scope(effective_params) if explicit_api_key and explicit_api_base: - return (explicit_api_base, explicit_api_key) - route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) - if route is None or route.provider not in _SINGLE_VENDOR_PROVIDERS: + return _EncryptionBoundary( + explicit_api_base, explicit_api_key, route.model if route else None, account_scope, explicit_pair=True + ) + if route is None: return None api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route) if not api_key: @@ -260,7 +282,7 @@ class EncryptedContentAffinityCheck(CustomLogger): ) if not api_base: return None - return (api_base, api_key, route.model, EncryptedContentAffinityCheck._account_scope(effective_params)) + return _EncryptionBoundary(api_base, api_key, route.model, account_scope, explicit_pair=False) @staticmethod def _params_mapping(litellm_params: object) -> Mapping[str, object] | None: @@ -356,7 +378,10 @@ class EncryptedContentAffinityCheck(CustomLogger): if boundary is None: return [], originating matches: Final = [ - d for d in healthy_deployments if self._encryption_boundary_key(d.get("litellm_params", {})) == boundary + d + for d in healthy_deployments + if (candidate := self._encryption_boundary_key(d.get("litellm_params", {}))) is not None + and boundary.is_shared_with(candidate) ] return matches, originating @@ -384,8 +409,8 @@ class EncryptedContentAffinityCheck(CustomLogger): ) @cache - def target_boundaries() -> frozenset[_EncryptionBoundary]: - return frozenset( + def target_boundaries() -> tuple[_EncryptionBoundary, ...]: + return tuple( boundary for target in target_deployments if (boundary := EncryptedContentAffinityCheck._encryption_boundary_key(target.get("litellm_params"))) @@ -409,7 +434,9 @@ class EncryptedContentAffinityCheck(CustomLogger): if origin is not None else None ) - return origin_boundary is not None and origin_boundary in target_boundaries() + return origin_boundary is not None and any( + origin_boundary.is_shared_with(target_boundary) for target_boundary in target_boundaries() + ) def should_strip_input_item(item: Mapping[str, object]) -> bool: return not target_can_decrypt(EncryptedContentAffinityCheck._model_id_of_input_item(item)) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index edd3b6d6e1d..e14a00499ed 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1103,14 +1103,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance(): plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params) assert pydantic_key is not None - assert ( - pydantic_key - == plain_key - == ( - "https://mateo-resource.openai.azure.com", - "fake-azure-resource-key-a", - ) - ) + assert pydantic_key == plain_key def test_boundary_key_resolves_missing_values_from_named_credential(): @@ -1134,7 +1127,13 @@ def test_boundary_key_resolves_missing_values_from_named_credential(): ): boundary = EncryptedContentAffinityCheck._encryption_boundary_key({"litellm_credential_name": "account-a"}) - assert boundary == ("https://account-a.example.com", "credential-key-a") + assert boundary is not None + assert boundary == EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "https://account-a.example.com", "api_key": "credential-key-a"} + ) + assert boundary != EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "https://account-a.example.com", "api_key": "credential-key-b"} + ) @pytest.mark.parametrize( @@ -2988,6 +2987,42 @@ def test_hop_strip_keeps_accounts_that_sign_in_through_headers_apart( assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) +@pytest.mark.parametrize( + "environment_key, target_model, target_can_decrypt", + [ + pytest.param("shared-key", GATEWAY_BASE["model"], True, id="environment key equal to the explicit key"), + pytest.param("other-key", GATEWAY_BASE["model"], False, id="environment key other than the explicit key"), + pytest.param("shared-key", "openai/gpt-5.5-mini", False, id="equal keys serving another model"), + ], +) +def test_hop_strip_matches_an_explicit_key_and_the_environment_key_on_one_api_base( + monkeypatch: pytest.MonkeyPatch, environment_key: str, target_model: str, target_can_decrypt: bool +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setenv("OPENAI_API_KEY", environment_key) + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", {**GATEWAY_BASE, "api_key": "shared-key"}), + _default_endpoint_deployment("target", {**GATEWAY_BASE, "model": target_model}), + ] + ) + ctx: Final = RoutingContext(router) + reasoning: Final = _reasoning_input_minted_by("origin")[0] + original_reasoning: Final = dict(reasoning) + request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + ctx, request_input, None, (_deployment_of(ctx, "target"),), unmarked_origin=None + ) + + assert request_input[0] == (original_reasoning if target_can_decrypt else SUMMARY_ONLY_REASONING) + + class _SecretReadCounter(CustomSecretManager): def __init__(self) -> None: super().__init__(secret_manager_name="read-counter") From 252350e6d551a603e65cc6aeab7c2598a9e9e5da Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:02:01 -0700 Subject: [PATCH 09/10] fix(router): skip a deployment whose params cannot be read in boundary checks A named credential with a value of the wrong type, or an extra header whose name is not a string, made the boundary check raise and fail the whole request. That deployment now has no boundary, so the others still keep the reasoning they can decrypt. Two explicit api_base plus api_key pairs also compare without resolving the provider, so they read no secrets, as on main. --- .../encrypted_content_affinity_check.py | 45 ++++++++------ .../test_encrypted_content_affinity_check.py | 61 ++++++++++++++++++- 2 files changed, 88 insertions(+), 18 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index c8739c2ff2a..93d4a93bfe5 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -36,12 +36,12 @@ Safe to enable globally: - No cache required. """ -from collections.abc import Iterator, Mapping, Sequence -from dataclasses import dataclass -from functools import cache +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from functools import cache, partial from typing import TYPE_CHECKING, Final, Optional, cast -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_router_logger @@ -67,6 +67,7 @@ if TYPE_CHECKING: from litellm.router_api import RoutingContext _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[object, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) @@ -82,15 +83,16 @@ class _ProviderRoute: class _EncryptionBoundary: api_base: object api_key: object - model: str | None + model: Callable[[], str | None] = field(compare=False) account_scope: frozenset[tuple[str, object]] explicit_pair: bool def is_shared_with(self, other: "_EncryptionBoundary") -> bool: if (self.api_base, self.api_key) != (other.api_base, other.api_key): return False - both_explicit: Final = self.explicit_pair and other.explicit_pair - return both_explicit or (self.model, self.account_scope) == (other.model, other.account_scope) + if self.explicit_pair and other.explicit_pair: + return True + return self.account_scope == other.account_scope and self.model() == other.model() class EncryptedContentAffinityCheck(CustomLogger): @@ -261,17 +263,16 @@ class EncryptedContentAffinityCheck(CustomLogger): effective_params: Final = with_named_credential(params) explicit_api_key: Final = effective_params.get("api_key") explicit_api_base: Final = effective_params.get("api_base") - provider_route: Final = EncryptedContentAffinityCheck._provider_route(effective_params) - route: Final = ( - provider_route - if provider_route is not None and provider_route.provider in _SINGLE_VENDOR_PROVIDERS - else None - ) account_scope: Final = EncryptedContentAffinityCheck._account_scope(effective_params) if explicit_api_key and explicit_api_base: return _EncryptionBoundary( - explicit_api_base, explicit_api_key, route.model if route else None, account_scope, explicit_pair=True + explicit_api_base, + explicit_api_key, + cache(partial(EncryptedContentAffinityCheck._single_vendor_model, effective_params)), + account_scope, + explicit_pair=True, ) + route: Final = EncryptedContentAffinityCheck._single_vendor_route(effective_params) if route is None: return None api_key: Final = explicit_api_key or EncryptedContentAffinityCheck._default_api_key(route) @@ -282,7 +283,7 @@ class EncryptedContentAffinityCheck(CustomLogger): ) if not api_base: return None - return _EncryptionBoundary(api_base, api_key, route.model, account_scope, explicit_pair=False) + return _EncryptionBoundary(api_base, api_key, lambda: route.model, account_scope, explicit_pair=False) @staticmethod def _params_mapping(litellm_params: object) -> Mapping[str, object] | None: @@ -311,9 +312,19 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _header_mapping(headers: object) -> Mapping[str, object]: if isinstance(headers, Mapping): - return _REQUEST_KWARGS_ADAPTER.validate_python(headers) + return {str(name): value for name, value in _HEADERS_ADAPTER.validate_python(headers).items()} return {} + @staticmethod + def _single_vendor_route(litellm_params: Mapping[str, object]) -> _ProviderRoute | None: + route: Final = EncryptedContentAffinityCheck._provider_route(litellm_params) + return route if route is not None and route.provider in _SINGLE_VENDOR_PROVIDERS else None + + @staticmethod + def _single_vendor_model(litellm_params: Mapping[str, object]) -> str | None: + route: Final = EncryptedContentAffinityCheck._single_vendor_route(litellm_params) + return route.model if route is not None else None + @staticmethod def _provider_route(litellm_params: Mapping[str, object]) -> _ProviderRoute | None: model: Final = litellm_params.get("model") @@ -329,7 +340,7 @@ class EncryptedContentAffinityCheck(CustomLogger): provider_model, provider, _, provider_api_base = get_llm_provider( model=model, litellm_params=GenericLiteLLMParams.model_validate(dict(litellm_params)) ) - except BadRequestError: + except (BadRequestError, ValidationError): return None return _ProviderRoute(provider_model, provider, provider_api_base) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index e14a00499ed..a36d320fe51 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -15,6 +15,7 @@ The mechanism works without any cache and supports two encoding strategies: encrypted_content back to their original forms before sending to the upstream provider. """ +import datetime import time from collections.abc import Mapping from typing import Final @@ -3049,6 +3050,7 @@ class _SecretReadCounter(CustomSecretManager): pytest.param( {"model": "groq/llama-4-scout"}, "origin", id="reasoning from a provider with no default boundary" ), + pytest.param({**GATEWAY_BASE, "api_key": "shared-key"}, "origin", id="reasoning between explicit pairs"), ], ) def test_hop_strip_reads_no_secrets_when_no_default_boundary_needs_resolving( @@ -3082,6 +3084,60 @@ def test_hop_strip_reads_no_secrets_when_no_default_boundary_needs_resolving( assert secret_reads.reads == 0 +@pytest.mark.parametrize( + "unreadable_params", + [ + pytest.param( + {"model": "openai/gpt-5.5", "litellm_credential_name": "mistyped"}, + id="named credential with a value of the wrong type", + ), + pytest.param( + {**GATEWAY_BASE, "api_key": "shared-key", "extra_headers": {1: "one"}}, + id="extra header whose name is not a string", + ), + ], +) +def test_hop_strip_keeps_reasoning_a_target_can_decrypt_beside_a_target_with_unreadable_params( + monkeypatch: pytest.MonkeyPatch, unreadable_params: dict[str, object] +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="mistyped", + credential_values={"api_key": "shared-key", "api_version": datetime.date(2024, 10, 21)}, + credential_info={}, + ) + ], + ) + shared_pair: Final = {**GATEWAY_BASE, "api_key": "shared-key"} + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", shared_pair), + _default_endpoint_deployment("unreadable", unreadable_params), + _default_endpoint_deployment("target", shared_pair), + ] + ) + ctx: Final = RoutingContext(router) + reasoning: Final = _reasoning_input_minted_by("origin")[0] + original_reasoning: Final = dict(reasoning) + request_input: Final = [reasoning, {"role": "user", "content": "and sunsets?"}] + + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + ctx, + request_input, + None, + (_deployment_of(ctx, "unreadable"), _deployment_of(ctx, "target")), + unmarked_origin=None, + ) + + assert request_input[0] == original_reasoning + def test_boundary_key_puts_pydantic_and_plain_params_on_one_default_endpoint_boundary(): from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( EncryptedContentAffinityCheck, @@ -3089,9 +3145,12 @@ def test_boundary_key_puts_pydantic_and_plain_params_on_one_default_endpoint_bou from litellm.types.router import LiteLLM_Params pydantic_key: Final = EncryptedContentAffinityCheck._encryption_boundary_key(LiteLLM_Params(**OPENAI_SHARED_KEY)) + plain_key: Final = EncryptedContentAffinityCheck._encryption_boundary_key(dict(OPENAI_SHARED_KEY)) assert pydantic_key is not None - assert pydantic_key == EncryptedContentAffinityCheck._encryption_boundary_key(dict(OPENAI_SHARED_KEY)) + assert plain_key is not None + assert pydantic_key.is_shared_with(plain_key) + assert plain_key.is_shared_with(pydantic_key) def test_boundary_key_has_no_default_endpoint_for_a_model_no_provider_claims(): From b29af5365fd5697beddc52ad71e6efd68c47283e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:17:23 -0800 Subject: [PATCH 10/10] fix(router): ask a secret manager for default boundaries at most once a minute per deployment --- .../encrypted_content_affinity_check.py | 34 ++++++++++++++- .../test_encrypted_content_affinity_check.py | 43 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 93d4a93bfe5..a679e73f94e 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -36,9 +36,11 @@ Safe to enable globally: - No cache required. """ +import hashlib from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, field -from functools import cache, partial +from functools import cache, lru_cache, partial +from time import monotonic from typing import TYPE_CHECKING, Final, Optional, cast from pydantic import BaseModel, TypeAdapter, ValidationError @@ -58,6 +60,7 @@ from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployme from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param +from litellm.secret_managers.main import secret_manager_would_be_consulted from litellm.types.llms.openai import AllMessageValues from litellm.types.router import Deployment, GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -70,6 +73,8 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _HEADERS_ADAPTER: Final = TypeAdapter(Mapping[object, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) +_DEFAULT_ROUTE_SECRETS: Final = ("USE_LITELLM_PROXY", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE") +_SECRET_MANAGER_REFRESH_SECONDS: Final = 60 @dataclass(frozen=True, slots=True) @@ -95,6 +100,13 @@ class _EncryptionBoundary: return self.account_scope == other.account_scope and self.model() == other.model() +@dataclass(frozen=True, slots=True) +class _ParamsInRefreshWindow: + digest: str + window: int + params: Mapping[str, object] = field(compare=False) + + class EncryptedContentAffinityCheck(CustomLogger): """ Routes follow-up Responses API requests to the deployment that produced @@ -256,10 +268,30 @@ class EncryptedContentAffinityCheck(CustomLogger): ``isinstance(dict)`` guard would silently drop the latter from boundary matching and fall back to the full pool — i.e. trigger the exact ``invalid_encrypted_content`` failure this check exists to prevent. + + When a secret manager would be asked for the provider defaults, a deployment's boundary is + reused for up to a minute, so a run of pin misses does not ask it again for every candidate. """ params: Final = EncryptedContentAffinityCheck._params_mapping(litellm_params) if params is None: return None + if not any(secret_manager_would_be_consulted(name) for name in _DEFAULT_ROUTE_SECRETS): + return EncryptedContentAffinityCheck._resolve_encryption_boundary(params) + return EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window( + _ParamsInRefreshWindow( + hashlib.sha256(repr(params).encode()).hexdigest(), + int(monotonic() // _SECRET_MANAGER_REFRESH_SECONDS), + params, + ) + ) + + @staticmethod + @lru_cache(maxsize=1024) + def _encryption_boundary_in_refresh_window(key: _ParamsInRefreshWindow) -> _EncryptionBoundary | None: + return EncryptedContentAffinityCheck._resolve_encryption_boundary(key.params) + + @staticmethod + def _resolve_encryption_boundary(params: Mapping[str, object]) -> _EncryptionBoundary | None: effective_params: Final = with_named_credential(params) explicit_api_key: Final = effective_params.get("api_key") explicit_api_base: Final = effective_params.get("api_base") diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index a36d320fe51..419318d8b72 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -3201,3 +3201,46 @@ async def test_affinity_pins_to_the_same_key_peer_on_the_default_endpoint_when_t assert result == [peer] assert reasoning_input[0] == original_reasoning + + +@pytest.mark.asyncio +async def test_repeated_pins_to_a_default_endpoint_peer_ask_the_secret_manager_once(monkeypatch: pytest.MonkeyPatch): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router( + model_list=[ + _default_endpoint_deployment("origin", OPENAI_SHARED_KEY), + _default_endpoint_deployment("peer", OPENAI_SHARED_KEY), + _default_endpoint_deployment("other-org", {**OPENAI_SHARED_KEY, "api_key": "other-key"}), + ] + ) + ctx: Final = RoutingContext(router) + peer: Final = _deployment_of(ctx, "peer") + healthy_deployments: Final = [peer, _deployment_of(ctx, "other-org")] + secret_reads: Final = _SecretReadCounter() + monkeypatch.setattr(litellm, "secret_manager_client", secret_reads) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + EncryptedContentAffinityCheck._encryption_boundary_in_refresh_window.cache_clear() + affinity: Final = EncryptedContentAffinityCheck(ctx=ctx) + + async def pin_a_turn_minted_by_the_origin() -> tuple[list[dict[str, object]], Mapping[str, object]]: + reasoning_input: Final = _reasoning_input_minted_by("origin") + result: Final[list[dict[str, object]]] = await affinity.async_filter_deployments( + model="gpt-5.5", + healthy_deployments=list(healthy_deployments), + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input}, + ) + return result, reasoning_input[0] + + first_pin, first_reasoning = await pin_a_turn_minted_by_the_origin() + reads_for_the_first_pin: Final = secret_reads.reads + second_pin, second_reasoning = await pin_a_turn_minted_by_the_origin() + + assert first_pin == second_pin == [peer] + assert first_reasoning == second_reasoning == _reasoning_input_minted_by("origin")[0] + assert reads_for_the_first_pin > 0 + assert secret_reads.reads == reads_for_the_first_pin