diff --git a/litellm/litellm_core_utils/credential_accessor.py b/litellm/litellm_core_utils/credential_accessor.py index 95e36a64ef7..b2310894c3a 100644 --- a/litellm/litellm_core_utils/credential_accessor.py +++ b/litellm/litellm_core_utils/credential_accessor.py @@ -1,9 +1,15 @@ """Utils for accessing credentials.""" +from collections.abc import Mapping from typing import Final import litellm -from litellm.types.utils import CredentialItem +from litellm.types.router import holds_api_key_secret_reference +from litellm.types.utils import ( + API_KEY_SECRET_REFERENCE_REJECTION, + CredentialItem, + credential_api_key_secret_reference, +) class CredentialAccessor: @@ -21,7 +27,7 @@ class CredentialAccessor: """Safe accessor for credentials.""" credential: Final = CredentialAccessor.find_credential(credential_name) - return {} if credential is None else credential.credential_values.copy() + return {} if credential is None else dict(credential_api_key_secret_reference(credential.credential_values)) @staticmethod def upsert_credentials(credentials: list[CredentialItem]): @@ -38,3 +44,20 @@ class CredentialAccessor: break else: litellm.credential_list.append(credential) + + +def sends_api_key_secret_reference(params: Mapping[str, object]) -> bool: + named: Final = params.get("litellm_credential_name") + credential_values: Final = CredentialAccessor.get_credential_values(named) if isinstance(named, str) else {} + return holds_api_key_secret_reference({**credential_values, **params}) + + +def reject_api_key_secret_reference(params: Mapping[str, object]) -> None: + if holds_api_key_secret_reference(params): + raise ValueError(API_KEY_SECRET_REFERENCE_REJECTION) + named: Final = params.get("litellm_credential_name") + if isinstance(named, str) and holds_api_key_secret_reference(CredentialAccessor.get_credential_values(named)): + raise ValueError( + f"Rejected Request: litellm_credential_name={named!r} names a credential holding an oidc/ api_key " + "secret reference, which only a deployment a proxy admin configures can attach." + ) diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py index 0f833ccb4b8..03b27201398 100644 --- a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py @@ -27,6 +27,7 @@ from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +from litellm.types.utils import API_KEY_SECRET_REFERENCE_REJECTION, OIDC_SECRET_REFERENCE_PREFIX if TYPE_CHECKING: from litellm.router import Router @@ -226,6 +227,8 @@ def _resolve_advisor_credentials(advisor_tool: dict) -> tuple[str | None, str | return None, None api_key: Final[str | None] = advisor_tool.get("api_key") api_base: Final[str | None] = advisor_tool.get("api_base") + if isinstance(api_key, str) and api_key.startswith(OIDC_SECRET_REFERENCE_PREFIX): + raise litellm.BadRequestError(message=API_KEY_SECRET_REFERENCE_REJECTION, model=None, llm_provider=None) if api_base is None: return api_key, None if not api_key: diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 94cf7ff4dbe..7a05121ff2b 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -21,7 +21,7 @@ from litellm.constants import ( MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import ( SSRFError, @@ -36,7 +36,12 @@ from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_me from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, ) -from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, Deployment, server_owned_wif_fields_present +from litellm.types.router import ( + CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, + Deployment, + reject_server_owned_wif_fields, + server_owned_wif_fields_present, +) from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params @@ -246,16 +251,19 @@ reject_server_owned_wif_params = _reject_server_owned_wif_params _CREDENTIAL_VALUES: Final = TypeAdapter(dict[str, object]) _DECODED_JSON: Final = TypeAdapter(object) +_LIST_ENTRIES: Final = TypeAdapter(tuple[object, ...]) def reject_federated_credential_reference(body: Mapping[str, object]) -> None: - """Raise ``ValueError`` if a request body picks the federated identity by credential name. + """Raise ``ValueError`` if a request body picks a server-owned identity by credential name. ``load_credentials_from_list`` merges a named credential's values into the call, so a body naming a federated credential moves the token exchange onto that credential's federation rule and organization exactly as sending the federation fields inline would, which - ``reject_server_owned_wif_params`` already refuses. Attaching one is a deployment decision, so - it is refused with no client-side opt-in to relax it, the same as the inline form. + ``reject_server_owned_wif_params`` already refuses, and a body naming one whose ``api_key`` is + an ``oidc/`` secret reference gets the proxy to read that secret for a call it shapes. Attaching + either is a deployment decision, so it is refused with no client-side opt-in to relax it, the + same as the inline form. Only credentials already loaded into memory can be resolved here, which is every credential the proxy would resolve for the call itself: ``load_credentials_from_list`` reads the same list. @@ -271,8 +279,8 @@ def reject_federated_credential_reference(body: Mapping[str, object]) -> None: if wif_fields: raise ValueError( f"Rejected Request: litellm_credential_name={named!r} names a credential configured for " - f"workload identity federation or OAuth token exchange ({wif_fields[0]}), which a request body " - "cannot choose. " + f"workload identity federation or OAuth token exchange, or holding an oidc/ api_key secret " + f"reference ({wif_fields[0]}), which a request body cannot choose. " "A proxy admin attaches it to a deployment." ) @@ -685,19 +693,52 @@ def is_request_body_safe( _reject_client_chosen_federation(user_config_params, manages_deployments=manages_deployments) litellm_params: Final = _coerce_metadata_to_dict(request_body.get("litellm_params")) if litellm_params is not None: - _reject_client_chosen_federation(litellm_params, manages_deployments=manages_deployments) + _reject_request_credential_params(litellm_params, manages_deployments=manages_deployments) litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata")) if litellm_params_metadata is not None: _check_banned_params( - litellm_params_metadata, - general_settings, - llm_router, - model, - manages_deployments=manages_deployments, + litellm_params_metadata, general_settings, llm_router, model, manages_deployments=manages_deployments ) + for user_config_params in _user_config_litellm_params(request_body): # pyright: ignore[reportUnknownArgumentType] # request_body is an untyped dict + reject_api_key_secret_reference(user_config_params) return True +def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: + if manages_deployments: + reject_server_owned_wif_fields(litellm_params) + else: + reject_server_owned_wif_params(litellm_params) + reject_federated_credential_reference(litellm_params) + for nested_params in _nested_configs(litellm_params): + reject_api_key_secret_reference(nested_params) + + +def _user_config_litellm_params(request_body: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + user_config: Final = _coerce_metadata_to_dict(request_body.get("user_config")) + if user_config is None: + return () + entries: Final = _mappings_in(user_config.get("model_list")) + _mappings_in(user_config.get("search_tools")) + nested: Final = tuple( + litellm_params + for entry in entries + if (litellm_params := _coerce_metadata_to_dict(entry.get("litellm_params"))) is not None + ) + defaults: Final = _coerce_metadata_to_dict(user_config.get("default_litellm_params")) + return nested if defaults is None else (*nested, defaults) + + +def _nested_configs(params: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + nested for key in _NESTED_CONFIG_KEYS if (nested := _coerce_metadata_to_dict(params.get(key))) is not None + ) + + +def _mappings_in(value: object) -> tuple[Mapping[str, object], ...]: + entries: Final = _LIST_ENTRIES.validate_python(value) if isinstance(value, list) else () + return tuple(mapping for entry in entries if (mapping := _coerce_metadata_to_dict(entry)) is not None) + + def carries_client_side_overrides(request_body: object) -> bool: if not isinstance(request_body, dict): return False diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 984aed71b1c..e1a5fdd5a71 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -35,6 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders +from litellm.utils import resolve_api_key_secret_reference if TYPE_CHECKING: from prisma import types as prisma_types @@ -241,7 +242,11 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, ) except Exception: # noqa: BLE001 # get_llm_provider raises on a model it cannot map, which means nothing to probe return None - return None if resolved_api_base is None else (resolved_api_base, resolved_api_key) + if resolved_api_base is None: + return None + return resolved_api_base, resolve_api_key_secret_reference( + resolved_api_key, model=model, llm_provider=provider, upstream=credentials + ) async def litellm_executed_provider_for( diff --git a/litellm/proxy/common_utils/credential_hydration.py b/litellm/proxy/common_utils/credential_hydration.py index a4e858692e6..f6a6c682cf0 100644 --- a/litellm/proxy/common_utils/credential_hydration.py +++ b/litellm/proxy/common_utils/credential_hydration.py @@ -18,8 +18,8 @@ from litellm.repositories.credentials_repository import CredentialsRepository from litellm.router_utils.clientside_credential_handler import clientside_credential_keys from litellm.types.router import ( GenericLiteLLMParams, - server_owned_wif_fields_named, server_owned_wif_fields_present, + server_owned_wif_fields_written, ) from litellm.types.utils import CredentialItem, LlmProviders, server_owned_wif_litellm_params @@ -68,7 +68,7 @@ def decrypted_or_stored(key: str, value: str) -> str: return value if decrypted is None else decrypted -def _decrypted(db_credential: CredentialItem) -> CredentialItem: +def decrypted_credential(db_credential: CredentialItem) -> CredentialItem: """The stored credential with every value decrypted, leaving already-plaintext values alone.""" decrypted_values: Final = MappingProxyType( { @@ -99,7 +99,7 @@ async def hydrate_named_credential_authoritative( db_credential: Final = await CredentialsRepository(prisma_client).find_by_name(credential_name) if db_credential is None: return await hydrate_named_credential(credential_name, prisma_client) - return _decrypted(db_credential) + return decrypted_credential(db_credential) async def hydrate_named_credential( @@ -114,7 +114,7 @@ async def hydrate_named_credential( db_credential: Final = await CredentialsRepository(prisma_client).find_by_name(credential_name) if db_credential is None: return None - return _decrypted(db_credential) + return decrypted_credential(db_credential) async def named_credential_wif_fields( @@ -129,11 +129,17 @@ async def named_credential_wif_fields( refuses whenever either side says the credential is server-owned. """ matching: Final = tuple(c for c in litellm.credential_list if c.credential_name == credential_name) - in_memory: Final = tuple(chain.from_iterable(server_owned_wif_fields_named(c.credential_values) for c in matching)) + in_memory: Final = tuple( + chain.from_iterable(server_owned_wif_fields_written(c.credential_values) for c in matching) + ) if prisma_client is None: return in_memory db_credential: Final = await CredentialsRepository(prisma_client).find_by_name(credential_name) - stored: Final = () if db_credential is None else server_owned_wif_fields_named(db_credential.credential_values) + stored: Final = ( + () + if db_credential is None + else server_owned_wif_fields_written(decrypted_credential(db_credential).credential_values) + ) return tuple(dict.fromkeys(in_memory + stored)) @@ -165,7 +171,7 @@ async def effective_server_owned_wif_fields( touching the field. """ from_stored: Final = () if stored is None else server_owned_wif_fields_present(stored) - from_incoming: Final = () if incoming is None else server_owned_wif_fields_named(incoming.keys()) + from_incoming: Final = () if incoming is None else server_owned_wif_fields_written(incoming) from_credential: Final = tuple( chain.from_iterable( await asyncio.gather( diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index f51d71a4c20..953c944bfb2 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.credential_hydration import ( + decrypted_credential, hydrate_named_credential_authoritative, named_credential_wif_fields, stored_credential_provider, @@ -61,7 +62,7 @@ from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object from litellm.repositories.base_repository import is_unique_violation from litellm.repositories.credentials_repository import CredentialsRepository -from litellm.types.router import server_owned_wif_fields_named +from litellm.types.router import server_owned_wif_fields_named, server_owned_wif_fields_written from litellm.types.utils import CreateCredentialItem, CredentialItem from .user_provider_credentials import ( @@ -97,7 +98,7 @@ def _reject_non_admin_wif_fields( raise ProxyException( message=( f"Only proxy admins can change {wif_fields[0]!r}, a server-owned workload identity federation " - "or OAuth token exchange parameter." + "or OAuth token exchange parameter or oidc/ api_key secret reference." ), type=ProxyErrorTypes.auth_error.value, code=status.HTTP_403_FORBIDDEN, @@ -112,13 +113,13 @@ def _incoming_wif_fields(incoming_values: Mapping[str, object], credential: Upda names in ``credential_values_to_delete``, since dropping a federation field off the stored credential breaks every deployment referencing it just as installing one would redirect them. """ - return server_owned_wif_fields_named(incoming_values) + server_owned_wif_fields_named( + return server_owned_wif_fields_written(incoming_values) + server_owned_wif_fields_named( credential.credential_values_to_delete or () ) def _stored_wif_fields(stored_credential: CredentialItem) -> tuple[str, ...]: - return server_owned_wif_fields_named(stored_credential.credential_values) + return server_owned_wif_fields_written(decrypted_credential(stored_credential).credential_values) def _reject_overlapping_credential_values(credential: UpdateCredentialItem) -> None: @@ -317,7 +318,7 @@ async def create_credential( status_code=400, detail="Credential values are required. Unable to infer credential values from model ID.", ) - _reject_non_admin_wif_fields(server_owned_wif_fields_named(credential_values), user_api_key_dict) + _reject_non_admin_wif_fields(server_owned_wif_fields_written(credential_values), user_api_key_dict) _reject_non_admin_wif_fields( await named_credential_wif_fields(credential.credential_name, prisma_client), user_api_key_dict ) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 3c8dd7890fc..1e7f67392e1 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -8,6 +8,7 @@ import time import traceback from collections.abc import Iterable, Mapping, Sequence from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import Final, Literal, TypedDict import fastapi @@ -42,7 +43,6 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check - reject_server_owned_wif_params, ) from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -80,6 +80,8 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields +from litellm.types.router import reject_server_owned_wif_fields +from litellm.types.utils import AdminApiKeySecretReference, admin_api_key_secret_reference #### Health ENDPOINTS #### @@ -161,6 +163,16 @@ def _request_inherits_config_credentials( return not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS) +def _admin_api_key_from_request( + request_litellm_params: Mapping[str, object], call_params: Mapping[str, object] +) -> Mapping[str, object]: + """Only reached once the model auth check passed, which refuses an ``oidc/`` api_key from anyone but a proxy + admin.""" + marked: Final = admin_api_key_secret_reference({**call_params, "api_key": request_litellm_params.get("api_key")}) + api_key: Final = marked.get("api_key") + return MappingProxyType({"api_key": api_key} if isinstance(api_key, AdminApiKeySecretReference) else {}) + + def _config_base_for_health_check( config_params: Mapping[str, object], request_params: Mapping[str, object], @@ -2291,7 +2303,7 @@ async def test_model_connection( ) selected_by_id: Final = deployment_by_id is not None - reject_server_owned_wif_params(request_litellm_params) + reject_server_owned_wif_fields(request_litellm_params) # Merge: config params (from proxy config) as base, request params override litellm_params = { **_config_base_for_health_check( @@ -2341,6 +2353,10 @@ async def test_model_connection( ) health_check_params: Final = { **_OBJECT_MAPPING.validate_python(litellm_params), + **_admin_api_key_from_request( + _OBJECT_MAPPING.validate_python(request_litellm_params), + _OBJECT_MAPPING.validate_python(litellm_params), + ), "secret_fields": SecretFields(raw_headers=RedactedDict(raw_headers)), } diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index dbeda4e27e3..54882288b49 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -254,8 +254,10 @@ from litellm.router import Router from litellm.secret_managers.main import get_secret_bool from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS +from litellm.types.router import holds_api_key_secret_reference from litellm.types.services import ServiceTypes from litellm.types.utils import ( + API_KEY_SECRET_REFERENCE_REJECTION, CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR, CustomPricingLiteLLMParams, @@ -2735,6 +2737,9 @@ async def add_litellm_data_to_request( ), ) + if holds_api_key_secret_reference(data): # pyright: ignore[reportUnknownArgumentType] # data is an untyped dict + raise HTTPException(status_code=400, detail=API_KEY_SECRET_REFERENCE_REJECTION) + ## ENFORCED PARAMS CHECK # loop through each enforced param # example enforced_params ['user', 'metadata', 'metadata.generation_name'] diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7eae193dd8a..fc0868a5fc9 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -157,6 +157,7 @@ from litellm.types.router import ( ) from litellm.types.utils import ( COST_MAP_LOOKUP_KEY, + OIDC_SECRET_REFERENCE_PREFIX, echoed_cost_map_fields, echoed_cost_map_pricing_fields, without_server_derived_pricing, @@ -2298,7 +2299,8 @@ class ModelManagementAuthChecks: raise ProxyException( message=( f"Only proxy admins can change the credentials of a deployment configured for " - f"workload identity federation or OAuth token exchange ({wif_fields[0]!r})." + f"workload identity federation or OAuth token exchange, or holding an oidc/ api_key secret " + f"reference ({wif_fields[0]!r})." ), type=ProxyErrorTypes.auth_error.value, code=status.HTTP_403_FORBIDDEN, @@ -2315,6 +2317,33 @@ class ModelManagementAuthChecks: param="litellm_credential_name", ) + @staticmethod + def _reject_team_inline_api_key_secret_reference( + *, + model_params: Deployment, + incoming_params: Mapping[str, object] | None, + incoming_model_params: updateDeployment | None, + ) -> None: + api_key: Final = None if incoming_params is None else incoming_params.get("api_key") + incoming_model_info: Final = None if incoming_model_params is None else incoming_model_params.model_info + team_id: Final = ( + incoming_model_info.team_id + if incoming_model_info is not None and incoming_model_info.team_id is not None + else model_params.model_info.team_id + ) + if team_id is None or not isinstance(api_key, str) or not api_key.startswith(OIDC_SECRET_REFERENCE_PREFIX): + return + raise ProxyException( + message=( + "A team model cannot hold an oidc/ api_key secret reference, because LiteLLM resolves one only on a " + "proxy-level model or in a stored credential. Store the reference in a credential and set " + "litellm_credential_name on this model." + ), + type=ProxyErrorTypes.bad_request_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="api_key", + ) + @staticmethod async def can_user_make_model_call( model_params: Deployment, @@ -2332,6 +2361,9 @@ class ModelManagementAuthChecks: LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, ): raise HTTPException(status_code=403, detail="View-only users cannot manage models.") + ModelManagementAuthChecks._reject_team_inline_api_key_secret_reference( + model_params=model_params, incoming_params=incoming_params, incoming_model_params=incoming_model_params + ) # Federation fields choose which server-side secret is read and where the org-scoped token # it buys is sent, so only a proxy admin may point a federated deployment somewhere else. # Evaluated on the RESULTING deployment: a patch attaching a credential by name inherits diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 3d1be882779..270a1cc2f3b 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -10,7 +10,11 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i LiteLLM_ManagedVectorStore, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.secret_managers.main import get_secret_str +from litellm.secret_managers.main import ( + UnresolvableSecretReference, + get_secret_str, + resolve_local_oidc_reference, +) from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict @@ -39,22 +43,17 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str | return credentials -# Network I/O prohibited; get_credentials() is synchronous inside async routes. -# oidc/env_path/ is excluded because it bypasses the oidc/file/ credential-directory allowlist. -_LOCAL_OIDC_PREFIXES: Final = ("oidc/file/", "oidc/env/") - - def _resolve_oidc_reference(credential: str | None) -> str | None: - """Resolve local ``oidc/...`` credentials on every call to pick up rotated tokens.""" - if credential is None or not credential.startswith(_LOCAL_OIDC_PREFIXES): - return credential - try: - return get_secret_str(credential) - except (OSError, ValueError) as exc: - raise HTTPException( - status_code=401, - detail=f"pass-through credential {credential} could not be resolved: {exc}", - ) from exc + if credential is None: + return None + match resolve_local_oidc_reference(credential): + case UnresolvableSecretReference(kind=kind, reason=reason): + raise HTTPException( + status_code=401, + detail=f"pass-through credential {kind} secret reference could not be resolved: {reason}", + ) + case str() as resolved: + return resolved class PassthroughEndpointRouter: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5030b1512a6..689a5f38416 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -202,7 +202,7 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import cost_map_omits_token_price, load_credentials_from_list +from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential if TYPE_CHECKING: from aiohttp import ClientSession @@ -302,7 +302,11 @@ import litellm import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target -from litellm._logging import redact_string, verbose_proxy_logger, verbose_router_logger +from litellm._logging import ( + redact_string, + verbose_proxy_logger, + verbose_router_logger, +) from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import DeclaredBatchRead from litellm.caching.redis_batch import ( @@ -1350,6 +1354,7 @@ class _WebsearchInterceptionLitellmSettings(TypedDict, total=False): _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody) _LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody) _ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig) +_LITELLM_PARAMS_COPY: Final = TypeAdapter(dict[str, object]) _WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings) _UNTYPED_VALUE: Final = TypeAdapter(object) @@ -14712,6 +14717,20 @@ def _get_provider_token_counter( return None, None, None +def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: + call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params) + try: + load_credentials_from_list(call_params) + except litellm.AuthenticationError as e: + raise ProxyException( + message=e.message, + type="auth_error", + param="model", + code=401, + ) from e + return call_params + + _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -14771,7 +14790,12 @@ async def _try_provider_token_count( model_to_use=model_to_use or "", messages=messages, contents=contents, - deployment=deployment_in_region, + deployment=None + if deployment_in_region is None + else { + **deployment_in_region, + "litellm_params": _with_resolved_api_key(deployment_in_region.get("litellm_params", {})), + }, request_model=request_model, tools=tools, system=system, @@ -15207,7 +15231,12 @@ async def count_request_tokens( if deployment is not None: litellm_model_name = deployment.get("litellm_params", {}).get("model") model_info = deployment.get("model_info", {}) - load_credentials_from_list(deployment.get("litellm_params", {})) + deployment = { # rebind-ok: merge the named credential into a copy + **deployment, + "litellm_params": with_named_credential( + _LITELLM_PARAMS_COPY.validate_python(deployment.get("litellm_params", {})) + ), + } # remove the custom_llm_provider_prefix in the litellm_model_name if "/" in litellm_model_name: litellm_model_name = litellm_model_name.split("/", 1)[1] diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index ea47d744cae..088dfb54155 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -799,12 +799,17 @@ async def realtime_health_check( import websockets resolved_params: Final = _model_params_with_stored_credentials(model_params or _EMPTY_MODEL_PARAMS) - resolved_api_key: Final = cast( # cast-ok: provider parameters expose optional string credentials - str | None, api_key or resolved_params.get("api_key") - ) resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints str | None, api_base or resolved_params.get("api_base") ) + resolved_api_key: Final = resolve_api_key_secret_reference( + cast( # cast-ok: provider parameters expose optional string credentials + str | None, api_key or resolved_params.get("api_key") + ), + model=model, + llm_provider=custom_llm_provider, + upstream=MappingProxyType({**resolved_params, "api_base": resolved_api_base}), + ) resolved_api_version: Final = cast( # cast-ok: provider parameters expose optional string versions str | None, api_version or resolved_params.get("api_version") ) diff --git a/litellm/router.py b/litellm/router.py index d8da69f7789..5a34a00e204 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_parent_otel_span_from_kwargs, ) from litellm.litellm_core_utils.coroutine_checker import coroutine_checker -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, @@ -214,6 +214,7 @@ from litellm.router_utils.common_utils import ( truncate_fallback_error_detail, warn_on_ignored_model_capability_flags, warn_on_provider_credential_mismatch, + with_admin_api_key_secret_reference, without_model_capability_flags, without_router_only_kwargs, ) @@ -413,6 +414,7 @@ from litellm.utils import ( is_generalized_model_info, is_region_allowed, provider_rejectable_params, + resolve_api_key_secret_reference, set_live_deployment_replay, ) @@ -10073,7 +10075,7 @@ class Router: f"Invalid regex in tag_regex for model '{deployment.model_name}': {pattern!r} — {exc}" ) from exc - deployment = self._add_deployment(deployment=deployment) + deployment = self._add_deployment(deployment=with_admin_api_key_secret_reference(deployment)) model: Final = deployment.to_json(exclude_none=True) @@ -10841,7 +10843,7 @@ class Router: declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params) - served: Final = without_model_capability_flags(deployment) + served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) # add to model list _deployment: Final = served.to_json(exclude_none=True) @@ -11414,7 +11416,12 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return - api_key: Final = params.api_key or dynamic_api_key + api_key: Final = resolve_api_key_secret_reference( + params.api_key or dynamic_api_key, + model=model, + llm_provider=provider, + upstream=MappingProxyType({"api_base": params.api_base}), + ) headers: Final = TypeAdapter(Mapping[str, str]).validate_python( params.get("extra_headers") or params.get("headers") or MappingProxyType({}) ) @@ -14604,6 +14611,7 @@ class Router: back on a routing pass that pins none.""" if not tier_litellm_params: return False + reject_api_key_secret_reference(tier_litellm_params) accepted_tier_params: Final = self._tier_params_the_target_accepts(model, tier_litellm_params, request_kwargs) surface_tier_params: Final = self._tier_ceiling_under_the_surface_name( accepted_tier_params, responses_call=responses_call diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index 65980449937..2bd58405693 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -14,6 +14,8 @@ Ensures cooldowns are applied correctly. from collections.abc import MutableMapping from typing import Final, cast # noqa: TID251 # narrows the untyped request dict for mutation +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.credential_accessor import sends_api_key_secret_reference from litellm.types.utils import server_owned_wif_litellm_params clientside_credential_keys: Final = ["api_key", "api_base", "base_url"] @@ -115,6 +117,16 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di if "api_base" in request_kwargs or "base_url" in request_kwargs: from litellm.llms.github_copilot.per_user_auth import github_copilot_auth_mode + if sends_api_key_secret_reference(litellm_params): # pyright: ignore[reportUnknownArgumentType] # litellm_params is the untyped request dict + raise BadRequestError( + message=( + "Rejected Request: this deployment's api_key is an oidc/ secret reference, so a request cannot " + "send it to an api_base of its own. Send your own api_key with the api_base." + ), + model=None, + llm_provider=None, + ) + # A per-user OAuth credential must survive the clear: dropping # litellm_credential_name / github_copilot_auth_type would flip the call to # shared mode and send the admin's Copilot token to the redirected host. diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 9b9a3d185cc..caab6e572de 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -23,7 +23,12 @@ from litellm.types.router import ( LiteLLM_Params, SkippedFallbackTarget, ) -from litellm.types.utils import LlmProviders, ProviderSpecificModelInfo +from litellm.types.utils import ( + AdminApiKeySecretReference, + LlmProviders, + ProviderSpecificModelInfo, + admin_api_key_secret_reference, +) _V = TypeVar("_V") @@ -32,6 +37,19 @@ ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_er _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment: + if deployment.model_info.get("team_id") is not None or isinstance( + deployment.litellm_params.api_key, AdminApiKeySecretReference + ): + return deployment + params: Final = admin_api_key_secret_reference(deployment.litellm_params.model_dump(exclude_none=True)) + if not isinstance(params.get("api_key"), AdminApiKeySecretReference): + return deployment + return deployment.model_copy( + update={"litellm_params": deployment.litellm_params.model_copy(update={"api_key": params["api_key"]})} + ) + + def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index bc870572f38..e1201df1a13 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -13,6 +13,7 @@ import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy +from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.router_utils.add_retry_fallback_headers import ( @@ -816,6 +817,7 @@ async def run_async_fallback( for target in fallback_model_group: if isinstance(target, dict): reject_server_owned_wif_params(target) + reject_api_key_secret_reference(target) for mg in fallback_model_group: if mg == failed_model_group: diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index d12e200772f..f4234e26c60 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -4,6 +4,7 @@ import os import time import traceback from collections.abc import Callable +from dataclasses import dataclass from functools import lru_cache from typing import Final @@ -89,7 +90,7 @@ def _resolve_oidc_file_path(requested_path: str) -> str: if not os.path.isabs(requested_path): raise OidcPathNotAllowedError( "oidc/file path must be absolute. Use the format " - "'oidc/file//var/run/secrets/' (note the leading slash " + "'oidc/file//' (note the leading slash " "after 'oidc/file/')." ) resolved: Final = os.path.realpath(requested_path) @@ -164,6 +165,59 @@ def get_secret_str( return value +LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/") + + +@dataclass(frozen=True, slots=True) +class UnresolvableSecretReference: + kind: str + reason: str + + +_OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"}) + + +def oidc_reference_kind(reference: str) -> str: + provider: Final = reference.removeprefix("oidc/").split("/", 1)[0] + return f"oidc/{provider}" if provider in _OIDC_PROVIDERS else "oidc/" + + +def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReference: + """Only local kinds resolve, since request paths must not do network I/O, and ``oidc/env_path/`` is + excluded because it bypasses the ``oidc/file/`` credential-directory allowlist.""" + if not value.startswith(LOCAL_OIDC_SECRET_PREFIXES): + return value + kind: Final = oidc_reference_kind(value) + try: + secret: Final = get_secret_str(value) + except (OSError, ValueError) as exc: + return _logged_unresolvable(value, kind, _unresolvable_reason(kind, exc)) + if not secret: + return _logged_unresolvable(value, kind, "it resolved to an empty value") + return secret + + +def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference: + verbose_logger.warning( + "Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason + ) + return UnresolvableSecretReference(kind=kind, reason=reason) + + +def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str: + match exc: + case UnicodeDecodeError(): + return "the file is not UTF-8 text" + case OidcPathNotAllowedError(): + return str(exc) + case OSError(strerror=str() as strerror): + return strerror + case _ if kind == "oidc/env": + return "the environment variable is not set" + case _: + return "the file could not be read" + + def normalize_nonempty_secret_str(val: str | None) -> str | None: """ Strip whitespace and treat None, '', and whitespace-only strings as unset. @@ -213,7 +267,7 @@ def get_secret( # Example: oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/stability.stable-diffusion-xl-v1/invoke if secret_name.startswith("oidc/"): - secret_name_split: Final = secret_name.replace("oidc/", "") + secret_name_split: Final = secret_name.removeprefix("oidc/") oidc_provider, oidc_aud = secret_name_split.split("/", 1) oidc_aud = "/".join(secret_name_split.split("/")[1:]) # TODO: Add caching for HTTP requests diff --git a/litellm/types/router.py b/litellm/types/router.py index f83127806db..36d298b2bc2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -42,6 +42,16 @@ from .litellm_params import RoutingStrategyName from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject from .search import SearchProvider +from .utils import ( + # private alias: `from .types.router import *` would rebind a public Final in litellm/__init__.py + API_KEY_SECRET_REFERENCE_REJECTION as _API_KEY_SECRET_REFERENCE_REJECTION, +) +from .utils import ( + OIDC_SECRET_REFERENCE_PREFIX as _OIDC_SECRET_REFERENCE_PREFIX, +) +from .utils import ( + AdminApiKeySecretReference as _AdminApiKeySecretReference, +) from .utils import ( CooldownDeploymentInfo, CustomPricingLiteLLMParams, @@ -50,7 +60,6 @@ from .utils import ( StandardLoggingRoutingDecision, ) from .utils import ( - # private alias: `from .types.router import *` would rebind a public Final in litellm/__init__.py server_owned_wif_litellm_params as _server_owned_wif_litellm_params, ) @@ -362,7 +371,7 @@ class CredentialLiteLLMParams(LiteLLMBaseModel): def __init__(self, /, **data: object) -> None: ... # kwargs-ok: credential fields vary by provider - api_key: str | None = None + api_key: _AdminApiKeySecretReference | str | None = None api_base: str | None = None api_version: str | None = None ## AZURE OAUTH ## @@ -451,15 +460,29 @@ class CredentialLiteLLMParams(LiteLLMBaseModel): token_exchange_audience: str | None = None +def _api_key_secret_reference(fields: Mapping[str, object]) -> tuple[str, ...]: + """The call path reads an ``oidc/`` api_key reference from the proxy's own filesystem or environment + and sends the result upstream, so it is server-owned the same way a federation field is.""" + api_key: Final = fields.get("api_key") + return ("api_key",) if isinstance(api_key, str) and api_key.startswith(_OIDC_SECRET_REFERENCE_PREFIX) else () + + +def holds_api_key_secret_reference(fields: Mapping[str, object]) -> bool: + return bool(_api_key_secret_reference(fields)) + + def server_owned_wif_fields_present(fields: Mapping[str, object]) -> tuple[str, ...]: - """Server-owned federation or OAuth token-exchange field names set in ``fields``. + """Server-owned federation or OAuth token-exchange field names set in ``fields``, plus ``api_key`` + when it holds an ``oidc/`` secret reference. ``fields`` is a ``litellm_params`` dict (or a credential's ``credential_values`` mapping, which feeds the same resolution when referenced by name). Derived from ``server_owned_wif_litellm_params`` rather than hand-copied, so a persistence gate built on this stays correct when a server-owned field is added there. """ - return tuple(name for name in _server_owned_wif_litellm_params if fields.get(name) is not None) + return tuple( + name for name in _server_owned_wif_litellm_params if fields.get(name) is not None + ) + _api_key_secret_reference(fields) def server_owned_wif_fields_named(keys: Container[str]) -> tuple[str, ...]: @@ -475,6 +498,10 @@ def server_owned_wif_fields_named(keys: Container[str]) -> tuple[str, ...]: return tuple(name for name in _server_owned_wif_litellm_params if name in keys) +def server_owned_wif_fields_written(values: Mapping[str, object]) -> tuple[str, ...]: + return server_owned_wif_fields_named(values) + _api_key_secret_reference(values) + + _WIF_POINTER_FIELDS: Final = frozenset(name for name in _server_owned_wif_litellm_params if name.endswith("_ref")) @@ -1404,7 +1431,8 @@ class AdaptiveRouterPreferences(LiteLLMBaseModel): def reject_server_owned_wif_params(body: Mapping[str, object]) -> None: """Raise ``ValueError`` if a mapping that did not come from deployment config carries a - server-owned workload identity federation or OAuth token-exchange field. + server-owned workload identity federation or OAuth token-exchange field, or an ``api_key`` + holding an ``oidc/`` secret reference. These are never settable inline on a client surface, with or without a client-side credential opt-in. Naming a stored credential that already holds them is the other way in and has its own @@ -1413,6 +1441,14 @@ def reject_server_owned_wif_params(body: Mapping[str, object]) -> None: This lives here rather than under ``litellm.proxy`` so the router can call it on a post-authentication merge without core importing from the proxy package. """ + reject_server_owned_wif_fields(body) + if holds_api_key_secret_reference(body): + raise ValueError(_API_KEY_SECRET_REFERENCE_REJECTION) + + +def reject_server_owned_wif_fields(body: Mapping[str, object]) -> None: + """``reject_server_owned_wif_params`` without the ``oidc/`` api_key check, for the deployment writes whose + role gate already refuses a non-admin setting one.""" submitted_names: Final = frozenset(key.removesuffix("[]") for key in body) for param in _server_owned_wif_litellm_params: if param in submitted_names: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 41581d702f3..99788dad4e1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -13,6 +13,7 @@ from typing import ( TypeAlias, get_args, ) +from urllib.parse import urlsplit import httpx from openai._models import BaseModel as OpenAIObject @@ -39,6 +40,7 @@ from pydantic import ( ConfigDict, Field, FieldSerializationInfo, + GetCoreSchemaHandler, JsonValue, PrivateAttr, SkipValidation, @@ -46,6 +48,7 @@ from pydantic import ( field_validator, ) from pydantic.main import IncEx +from pydantic_core import core_schema from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._logging import verbose_logger @@ -4186,6 +4189,109 @@ oauth_token_exchange_litellm_params: Final = ( "token_exchange_scope", ) github_copilot_oauth_litellm_params: Final = ("github_copilot_auth_type",) +OIDC_SECRET_REFERENCE_PREFIX: Final = "oidc/" +API_KEY_SECRET_REFERENCE_REJECTION: Final = ( + "Rejected Request: api_key holds an oidc/ secret reference, which only a deployment or credential a proxy " + "admin configures can carry." +) + + +API_KEY_UPSTREAM_PARAMS: Final = ( + "api_base", + "base_url", + "aws_bedrock_runtime_endpoint", + "s3_endpoint_url", + "sagemaker_base_url", + "deployment_url", +) + + +_DEFAULT_PORTS: Final = MappingProxyType({"http": 80, "ws": 80, "https": 443, "wss": 443}) + + +def _origin(url: str) -> str: + text: Final = url.strip() + try: + parts: Final = urlsplit(text) + explicit_port: Final = parts.port + except ValueError: + return text.rstrip("/").lower() + host: Final = (parts.hostname or "").rstrip(".") + if not host: + return text.rstrip("/").lower() + scheme: Final = parts.scheme.lower() + port: Final = explicit_port if explicit_port is not None else _DEFAULT_PORTS.get(scheme) + return f"{scheme}://{host}" if port is None else f"{scheme}://{host}:{port}" + + +_URL_IN_MODEL: Final = re.compile(r"https?://\S+", re.IGNORECASE) + + +def _model_url(model: object) -> str | None: + found: Final = _URL_IN_MODEL.search(model) if isinstance(model, str) else None + return None if found is None else found.group(0) + + +def upstream_origins(params: Mapping[str, object]) -> frozenset[str]: + values: Final = (*(params.get(key) for key in API_KEY_UPSTREAM_PARAMS), _model_url(params.get("model"))) + return frozenset(_origin(value) for value in values if isinstance(value, str) and value) + + +class AdminApiKeySecretReference(str): + """An ``oidc/`` api_key reference read from a source only a proxy admin writes: a proxy-level deployment or a + stored credential. The call path swaps only this type for the secret it points to, so the same text arriving + in a request, a prompt template, or a row a non-admin wrote is sent as written, and the secret goes only to the + origins the admin configured beside the reference.""" + + def __new__(cls, reference: str, origins: frozenset[str]) -> "AdminApiKeySecretReference": + return super().__new__(cls, reference) + + def __init__(self, reference: str, origins: frozenset[str]) -> None: + super().__init__() + self.origins: Final = origins + + def __reduce__(self) -> tuple[type["AdminApiKeySecretReference"], tuple[str, frozenset[str]]]: + return type(self), (str(self), self.origins) + + def may_reach(self, upstream: Mapping[str, object]) -> bool: + destinations: Final = upstream_origins(upstream) + return destinations <= self.origins if destinations else not self.origins + + @classmethod + def __get_pydantic_core_schema__(cls, source: object, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + return core_schema.is_instance_schema(cls, serialization=core_schema.to_string_ser_schema()) + + +class CredentialApiKeySecretReference(AdminApiKeySecretReference): + """An admin ``oidc/`` api_key reference stored in a named credential, so its origins are the credential's own.""" + + +def _unmarked_api_key_secret_reference(params: Mapping[str, object]) -> str | None: + api_key: Final = params.get("api_key") + if ( + not isinstance(api_key, str) + or isinstance(api_key, AdminApiKeySecretReference) + or not api_key.startswith(OIDC_SECRET_REFERENCE_PREFIX) + ): + return None + return api_key + + +def admin_api_key_secret_reference(params: Mapping[str, object]) -> Mapping[str, object]: + api_key: Final = _unmarked_api_key_secret_reference(params) + if api_key is None: + return params + return {**params, "api_key": AdminApiKeySecretReference(api_key, upstream_origins(params))} + + +def credential_api_key_secret_reference(credential_values: Mapping[str, object]) -> Mapping[str, object]: + api_key: Final = _unmarked_api_key_secret_reference(credential_values) + if api_key is None: + return credential_values + reference: Final = CredentialApiKeySecretReference(api_key, upstream_origins(credential_values)) + return {**credential_values, "api_key": reference} + + server_owned_wif_litellm_params: Final = ( anthropic_wif_litellm_params + openai_wif_litellm_params diff --git a/litellm/utils.py b/litellm/utils.py index 2b7abae358b..59e0d2d8180 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -228,12 +228,15 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ( ABOVE_THRESHOLD_COST_KEY_PATTERN, + API_KEY_UPSTREAM_PARAMS, OPENAI_RESPONSE_HEADERS, + AdminApiKeySecretReference, CallTypes, ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, CostPerToken, + CredentialApiKeySecretReference, CredentialItem, CustomHuggingfaceTokenizer, Delta, @@ -261,6 +264,7 @@ from litellm.types.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, + credential_api_key_secret_reference, is_litellm_owned_kwarg, ) @@ -457,7 +461,13 @@ from litellm.llms.base_llm.completion.transformation import BaseTextCompletionCo from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig -from litellm.secret_managers.main import get_secret +from litellm.secret_managers.main import ( + LOCAL_OIDC_SECRET_PREFIXES, + UnresolvableSecretReference, + get_secret, + oidc_reference_kind, + resolve_local_oidc_reference, +) from ._logging import is_debugging_on, verbose_logger from .caching.caching import ( @@ -721,16 +731,13 @@ def get_applied_guardrails(kwargs: dict[str, object]) -> list[str]: return applied_guardrails -def load_credentials_from_list(kwargs: dict): - """ - Updates kwargs with the credentials if credential_name in kwarg - """ +def _named_credential_values(kwargs: Mapping[str, object]) -> Mapping[str, object]: # Access CredentialAccessor via module to trigger lazy loading if needed credential_accessor: Final[type[CredentialAccessor]] = litellm_utils.CredentialAccessor credential_name: Final = kwargs.get("litellm_credential_name") - if not credential_name: - return + if not isinstance(credential_name, str) or not credential_name: + return MappingProxyType({}) credential: Final = credential_accessor.find_credential(credential_name) if credential is None: verbose_logger.warning( @@ -738,10 +745,93 @@ def load_credentials_from_list(kwargs: dict): credential_name, len(litellm.credential_list), ) - return - for key, value in credential.credential_values.items(): - if key not in kwargs: - kwargs[key] = value + return MappingProxyType({}) + return MappingProxyType( + { + key: value + for key, value in credential_api_key_secret_reference(credential.credential_values).items() + if key not in kwargs + } + ) + + +def resolve_api_key_secret_reference( + api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] +) -> str | None: + if not isinstance(api_key, AdminApiKeySecretReference): + return api_key + return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream) + + +def _secret_behind_api_key_reference( + reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object] +) -> str: + if not reference.may_reach(upstream): + raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model) + if not reference.startswith(LOCAL_OIDC_SECRET_PREFIXES): + raise _secret_reference_error( + f"api_key is an {oidc_reference_kind(reference)} secret reference, a kind api_key does not support. " + "Use oidc/file/ or oidc/env/.", + llm_provider=llm_provider, + model=model, + ) + match resolve_local_oidc_reference(reference): + case UnresolvableSecretReference(kind=kind, reason=reason): + raise _secret_reference_error( + f"api_key {kind} secret reference could not be resolved: {reason}", + llm_provider=llm_provider, + model=model, + ) + case str() as secret: + return secret + + +def _secret_reference_error(message: str, llm_provider: object, model: object) -> AuthenticationError: + return AuthenticationError(message=message, llm_provider=llm_provider, model=model) + + +def _unreachable_upstream_message(reference: AdminApiKeySecretReference) -> str: + if isinstance(reference, CredentialApiKeySecretReference): + return ( + "The api_key secret reference comes from a stored credential, so LiteLLM sends it only to " + "that credential's own api_base. Set this api_base on the credential, or give this model its own " + "credential." + ) + return ( + "The api_key secret reference goes only to the host its deployment configures, and this call " + "points at another one. Send your own api_key with that host." + ) + + +def with_named_credential(kwargs: Mapping[str, object]) -> Mapping[str, object]: + return {**kwargs, **_named_credential_values(kwargs)} + + +def load_credentials_from_list(kwargs: dict[str, object]) -> None: + """ + Updates kwargs with the credentials if credential_name in kwarg, then swaps an admin-configured oidc/ api_key + reference for the secret it points to, read now so a rotated token is used on the next call + """ + kwargs.update(_credential_updates(kwargs)) + + +def _credential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]: + credential_values: Final = _named_credential_values(kwargs) + api_key: Final = kwargs.get("api_key", credential_values.get("api_key")) + if not isinstance(api_key, AdminApiKeySecretReference): + return credential_values + secret: Final = _secret_behind_api_key_reference( + api_key, + model=kwargs.get("model"), + llm_provider=kwargs.get("custom_llm_provider"), + upstream=MappingProxyType( + { + "model": kwargs.get("model"), + **{key: kwargs.get(key, credential_values.get(key)) for key in API_KEY_UPSTREAM_PARAMS}, + } + ), + ) + return MappingProxyType({**credential_values, "api_key": secret}) def get_dynamic_callbacks( @@ -8150,6 +8240,23 @@ def _infer_valid_provider_from_env_vars( return valid_providers +def _provider_listing_params( + custom_llm_provider: str, litellm_params: LiteLLM_Params | None +) -> Mapping[str, object] | None: + if litellm_params is None: + return None + params: Final = litellm_params.model_dump(exclude_none=True) + api_key: Final = litellm_params.api_key + if not isinstance(api_key, AdminApiKeySecretReference): + return params + return { + **params, + "api_key": _secret_behind_api_key_reference( + api_key, model=litellm_params.model, llm_provider=custom_llm_provider, upstream=params + ), + } + + def _get_valid_models_from_provider_api( provider_config: BaseLLMModelInfo, custom_llm_provider: str, @@ -8161,7 +8268,7 @@ def _get_valid_models_from_provider_api( if cached_result is not None: return cached_result models: Final = provider_config.discover_models( - litellm_params=litellm_params.model_dump(exclude_none=True) if litellm_params is not None else None + litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) ) _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) diff --git a/litellm/vector_store_files/main.py b/litellm/vector_store_files/main.py index c03963980de..792157b09ce 100644 --- a/litellm/vector_store_files/main.py +++ b/litellm/vector_store_files/main.py @@ -23,7 +23,7 @@ from litellm.types.vector_store_files import ( VectorStoreFileObject, VectorStoreFileUpdateRequest, ) -from litellm.utils import ProviderConfigManager, client +from litellm.utils import ProviderConfigManager, client, load_credentials_from_list from litellm.vector_store_files.utils import VectorStoreFileRequestUtils base_llm_http_handler = BaseLLMHTTPHandler() @@ -49,6 +49,7 @@ def _prepare_registry_credentials( kwargs.update(registry_credentials) except Exception: pass + load_credentials_from_list(kwargs) @client diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 4d945310293..8b33b1ce12c 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -31,7 +31,7 @@ from litellm.types.vector_stores import ( VectorStoreSearchResponse, VectorStoreSearchResult, ) -from litellm.utils import ProviderConfigManager, client +from litellm.utils import ProviderConfigManager, client, load_credentials_from_list from litellm.vector_stores.utils import VectorStoreRequestUtils if TYPE_CHECKING: @@ -403,6 +403,7 @@ def search( kwargs.update(registry_credentials) except Exception: pass + load_credentials_from_list(kwargs) # get llm provider logic litellm_params: Final = GenericLiteLLMParams(vector_store_id=vector_store_id, **kwargs) diff --git a/tests/integration/security/test_api_key_secret_reference.py b/tests/integration/security/test_api_key_secret_reference.py new file mode 100644 index 00000000000..0b1187d2e0e --- /dev/null +++ b/tests/integration/security/test_api_key_secret_reference.py @@ -0,0 +1,783 @@ +from __future__ import annotations + +import json +import uuid +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.openai_wire import chat_reply, openai_error, responses_reply +from integration._support.process import ( + OwnedProxy, + WorkerHealthcheck, + graceful_stop_seconds, + owned_proxy_process, +) +from integration._support.wire import Reply, Request, wire_server +from integration.security._sinks import Recorder +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120)) + +_CLOSE: Final = MappingProxyType({"Connection": "close"}) +_FRESH: Final = httpx.Limits(max_keepalive_connections=0) +_CHAT_MODEL: Final = "openai/gpt-4o-mini" +_EMBEDDING_MODEL: Final = "openai/text-embedding-3-small" +_REALTIME_MODEL: Final = "openai/gpt-4o-realtime-preview" +_LISTED_MODEL: Final = "wif-listed-model" +_VECTOR_STORE: Final = "vs_wif_reference" +_SEARCH_PATH: Final = f"/v1/vector_stores/{_VECTOR_STORE}/search" +_BATCH_DEPLOYMENT: Final = "wif-vllm-batch" +_BEDROCK_MODEL: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" +_PROVIDER_INPUT_TOKENS: Final = 7919 +_UNSUPPORTED_REFERENCE: Final = "oidc/google/https://example.invalid" +_RUN_SECRETS_TOKEN: Final = "/run/secrets/wif-absent-token" +_SOURCES: Final = ("config_credential", "config_inline", "db_credential", "db_inline") +_UNREADABLE: Final = MappingProxyType( + {"missing_file": "wif-missing", "outside_allowlist": "wif-outside", "unsupported_kind": "wif-unsupported"} +) +_ATTACKS: Final = ( + "client_api_base_inline", + "client_api_base_credential", + "body_reference", + "body_credential_name", + "user_config", +) +_CREDENTIAL_ROW: Final = ( + 'SELECT credential_values::text AS stored, updated_at::text AS updated_at FROM "LiteLLM_CredentialsTable" ' + "WHERE credential_name = %s" +) +_MODEL_ROW: Final = ( + 'SELECT litellm_params::text AS stored, updated_at::text AS updated_at FROM "LiteLLM_ProxyModelTable" ' + "WHERE model_id = %s" +) +_ENDPOINTS: Final = MappingProxyType( + {"embeddings": ("/v1/embeddings", "wif-embedding"), "responses": ("/v1/responses", "wif-config-inline")} +) +_TEAM_WRITES: Final = MappingProxyType( + { + ("team_admin", "plain_key"): 200, + ("team_admin", "names_credential"): 403, + ("team_admin", "inline_reference"): 400, + ("proxy_admin", "inline_reference"): 400, + } +) +_MODEL_ROWS: Final = ( + "SELECT model_id FROM \"LiteLLM_ProxyModelTable\" WHERE model_info::jsonb ->> 'team_public_model_name' = %s" +) + + +@dataclass(frozen=True, slots=True) +class _Secrets: + allowed: Path + token: Path + missing: Path + outside: Path + + def current(self) -> str: + return self.token.read_text() + + def rotate(self) -> str: + value: Final = f"wif-token-{uuid.uuid4().hex}" + self.token.write_text(value) + return value + + def overrides(self) -> Mapping[str, str]: + return MappingProxyType( + {"LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS": str(self.allowed), "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"} + ) + + +def _reference(path: Path) -> str: + return f"oidc/file/{path}" + + +def _secrets(directory: Path) -> _Secrets: + allowed: Final = directory / "secrets" + outside_dir: Final = directory / "outside" + allowed.mkdir() + outside_dir.mkdir() + secrets: Final = _Secrets( + allowed=allowed, token=allowed / "token", missing=allowed / "absent-token", outside=outside_dir / "token" + ) + secrets.rotate() + secrets.outside.write_text(f"outside-secret-{uuid.uuid4().hex}") + return secrets + + +def _json_reply(body: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _served(request: Request) -> Reply: + path: Final = request.target.split("?", 1)[0] + match request.method, path: + case "POST", "/v1/chat/completions": + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "gpt-4o-mini", "served", stream=False) + case "POST", "/v1/responses": + return responses_reply(f"resp_{uuid.uuid4().hex}", "gpt-4o-mini", "served", stream=False) + case "POST", "/v1/responses/input_tokens": + return _json_reply({"object": "response.input_tokens", "input_tokens": _PROVIDER_INPUT_TOKENS}) + case "POST", converse if converse.startswith("/model/") and converse.endswith("/converse"): + return _json_reply( + { + "output": {"message": {"role": "assistant", "content": [{"text": "served"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + } + ) + case "POST", "/v1/embeddings": + return _json_reply( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + } + ) + case "GET", "/v1/models": + return _json_reply( + {"object": "list", "data": [{"id": _LISTED_MODEL, "object": "model", "created": 0, "owned_by": "it"}]} + ) + case "POST", searched if searched == _SEARCH_PATH: + return _json_reply( + { + "object": "vector_store.search_results.page", + "search_query": "served", + "data": [], + "has_more": False, + "next_page": None, + } + ) + case _: + return openai_error(404) + + +def _provider(secrets: _Secrets) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.headers.get("authorization") != f"Bearer {secrets.current()}": + return openai_error(401) + return _served(request) + + return respond + + +def _intruder(request: Request) -> Reply: + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "gpt-4o-mini", "intercepted", stream=False) + + +def _deployment( + name: str, params: Mapping[str, JsonValue], info: Mapping[str, JsonValue] = MappingProxyType({}) +) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": dict(params), "model_info": dict(info)} + + +def _credential(name: str, api_key: str, api_base: str) -> dict[str, JsonValue]: + return { + "credential_name": name, + "credential_values": {"api_key": api_key, "api_base": api_base}, + "credential_info": {"custom_llm_provider": "openai"}, + } + + +def _named(model: str, credential: str) -> dict[str, JsonValue]: + return {"model": model, "litellm_credential_name": credential} + + +def _config(directory: Path, secrets: _Secrets, provider: str) -> Path: + inline: Final = {"api_key": _reference(secrets.token), "api_base": provider} + configuration: Final[dict[str, JsonValue]] = { + "model_list": [ + _deployment("wif-config-credential", _named(_CHAT_MODEL, "wif-config")), + _deployment("wif-config-inline", {"model": _CHAT_MODEL, **inline}), + _deployment("wif-embedding", _named(_EMBEDDING_MODEL, "wif-config"), {"mode": "embedding"}), + _deployment("wif-missing", _named(_CHAT_MODEL, "wif-missing")), + _deployment("wif-outside", _named(_CHAT_MODEL, "wif-outside")), + _deployment("wif-unsupported", _named(_CHAT_MODEL, "wif-unsupported")), + _deployment("wif-run-secrets", _named(_CHAT_MODEL, "wif-run-secrets")), + _deployment("wif-missing-fallback", _named(_CHAT_MODEL, "wif-missing")), + _deployment("wif-realtime", _named(_REALTIME_MODEL, "wif-config"), {"mode": "realtime"}), + _deployment("wif-vllm", {"model": "hosted_vllm/wif-local", **inline}, {"id": _BATCH_DEPLOYMENT}), + _deployment("openai/*", _named("openai/*", "wif-config")), + _deployment( + "wif-bedrock-endpoint", + { + "model": _BEDROCK_MODEL, + "aws_region_name": "us-east-1", + "api_key": _reference(secrets.token), + "aws_bedrock_runtime_endpoint": provider.removesuffix("/v1"), + }, + ), + _deployment( + "wif-mistral-passthrough", + { + "model": "mistral/mistral-small-latest", + "api_key": _reference(secrets.missing), + "use_in_pass_through": True, + }, + ), + ], + "credential_list": [ + _credential("wif-config", _reference(secrets.token), provider), + _credential("wif-missing", _reference(secrets.missing), provider), + _credential("wif-outside", _reference(secrets.outside), provider), + _credential("wif-unsupported", _UNSUPPORTED_REFERENCE, provider), + _credential("wif-run-secrets", f"oidc/file/{_RUN_SECRETS_TOKEN}", provider), + ], + "vector_store_registry": [ + { + "vector_store_name": "wif-vector-store", + "litellm_params": { + "vector_store_id": _VECTOR_STORE, + "custom_llm_provider": "openai", + "litellm_credential_name": "wif-config", + }, + } + ], + "router_settings": { + "fallbacks": [{"wif-missing-fallback": ["wif-config-credential"]}], + "disable_cooldowns": True, + }, + "litellm_settings": {"check_provider_endpoint": True}, + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "allow_client_side_credentials": True}, + } + path: Final = directory / "api_key_secret_reference_config.yaml" + path.write_text(yaml.safe_dump(configuration, sort_keys=False)) + return path + + +@dataclass(frozen=True, slots=True) +class _Stored: + credential: str + models: Mapping[str, str] + model_ids: Mapping[str, str] + + +def _delete_credential(gateway: Gateway, name: str) -> None: + response: Final = gateway.request("DELETE", f"/credentials/{name}") + assert response.status_code in (200, 404), response.text + + +def _created_model(gateway: Gateway, scenario: Scenario, body: Mapping[str, JsonValue]) -> tuple[str, str]: + created: Final = gateway.post("/model/new", body) + identity: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, identity) + return string_value(created["model_name"]), identity + + +def _stored(owned: OwnedProxy, scenario: Scenario, secrets: _Secrets, provider: str) -> _Stored: + credential: Final = f"db-oidc-{uuid.uuid4().hex}" + owned.gateway.post("/credentials", _credential(credential, _reference(secrets.token), provider)) + scenario.cleanups.callback(_delete_credential, owned.gateway, credential) + by_credential: Final = _created_model( + owned.gateway, + scenario, + _deployment(f"db-credential-{uuid.uuid4().hex}", _named(_CHAT_MODEL, credential)), + ) + inline: Final = _created_model( + owned.gateway, + scenario, + _deployment( + f"db-inline-{uuid.uuid4().hex}", + {"model": _CHAT_MODEL, "api_key": _reference(secrets.token), "api_base": provider}, + ), + ) + for name, identity in (by_credential, inline): + eventually(partial(owned.serves_only_on_every_worker, name, identity), bool, seconds=120) + return _Stored( + credential=credential, + models=MappingProxyType({"db_credential": by_credential[0], "db_inline": inline[0]}), + model_ids=MappingProxyType({"db_credential": by_credential[1], "db_inline": inline[1]}), + ) + + +@dataclass(frozen=True, slots=True) +class _Rig: + owned: OwnedProxy + provider: Recorder + intruder: Recorder + provider_base: str + intruder_base: str + secrets: _Secrets + stored: _Stored + boot: tuple[Request, ...] + boot_token: str + + def model(self, source: str) -> str: + match source: + case "config_credential": + return "wif-config-credential" + case "config_inline": + return "wif-config-inline" + case "db_credential" | "db_inline": + return self.stored.models[source] + case _: + pytest.fail(f"unknown source {source}") + + +def _is_discovery(request: Request) -> bool: + return (request.method, request.target) == ("GET", "/v1/models") + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("api_key_secret_reference").resolve() + secrets: Final = _secrets(directory) + with ( + gateway_from_environment() as gateway, + wire_server(_provider(secrets)) as provider_wire, + wire_server(_intruder) as intruder_wire, + ): + provider: Final = Recorder(provider_wire) + provider_base: Final = f"{provider_wire.url}/v1" + with owned_proxy_process( + gateway, + directory, + {**secrets.overrides(), "MISTRAL_API_BASE": provider_wire.url}, + config=_config(directory, secrets, provider_base), + workers=2, + healthcheck=WorkerHealthcheck.LENIENT, + ) as owned: + with owned.gateway.scenario() as scenario: + stored: Final = _stored(owned, scenario, secrets, provider_base) + boot: Final = eventually(provider.requests, lambda seen: any(_is_discovery(r) for r in seen)) + yield _Rig( + owned=owned, + provider=provider, + intruder=Recorder(intruder_wire), + provider_base=provider_base, + intruder_base=f"{intruder_wire.url}/v1", + secrets=secrets, + stored=stored, + boot=boot, + boot_token=secrets.current(), + ) + + +def _marker() -> str: + return f"wif-marker-{uuid.uuid4().hex}" + + +def _chat( + gateway: Gateway, + model: str, + marker: str, + extra: Mapping[str, JsonValue] = MappingProxyType({}), + *, + key: str | None = None, +) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"say {marker}"}], **extra}, + key=key, + headers=_CLOSE, + ) + + +def _carrying(requests: Sequence[Request], marker: str) -> tuple[Request, ...]: + return tuple(request for request in requests if marker in request.body.decode(errors="replace")) + + +def _bearers(requests: Sequence[Request]) -> tuple[str | None, ...]: + return tuple(request.headers.get("authorization") for request in requests) + + +def _anywhere(requests: Sequence[Request], value: str) -> bool: + return any( + value in json.dumps(dict(request.headers)) + request.body.decode(errors="replace") for request in requests + ) + + +def _reached_on_worker(rig: _Rig, model: str, idle: int) -> tuple[int, tuple[Request, ...]]: + marker: Final = _marker() + gateway: Final = rig.owned.gateway + with ( + rig.owned.suspended(idle), + httpx.Client(base_url=gateway.client.base_url, timeout=60, trust_env=False, limits=_FRESH) as client, + ): + response: Final = client.post( + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": f"say {marker}"}]}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + return response.status_code, _carrying(rig.provider.requests(), marker) + + +def _chat_on_worker(rig: _Rig, model: str, idle: int) -> tuple[int, tuple[str | None, ...]]: + status, reached = _reached_on_worker(rig, model, idle) + return status, _bearers(reached) + + +@pytest.mark.parametrize("source", _SOURCES) +def test_admin_reference_sends_the_current_token_from_every_worker(rig: _Rig, source: str) -> None: + first, second = sorted(eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2)) + model: Final = rig.model(source) + before: Final = rig.secrets.current() + sent_before: Final = (_chat_on_worker(rig, model, second), _chat_on_worker(rig, model, first)) + rotated: Final = rig.secrets.rotate() + sent_after: Final = (_chat_on_worker(rig, model, second), _chat_on_worker(rig, model, first)) + assert sent_before == ((200, (f"Bearer {before}",)),) * 2, sent_before + assert sent_after == ((200, (f"Bearer {rotated}",)),) * 2, sent_after + + +@pytest.mark.parametrize("endpoint", tuple(_ENDPOINTS)) +def test_other_call_types_send_the_current_token(rig: _Rig, endpoint: str) -> None: + marker: Final = _marker() + path, model = _ENDPOINTS[endpoint] + response: Final = rig.owned.gateway.request( + "POST", path, {"model": model, "input": f"say {marker}"}, headers=_CLOSE + ) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.secrets.current()}",)), response.text + + +@pytest.mark.parametrize("shape", ("credential_name", "inline")) +def test_connection_with_an_admin_reference_reaches_the_provider(rig: _Rig, shape: str) -> None: + params: Final[dict[str, JsonValue]] = ( + _named(_CHAT_MODEL, "wif-config") + if shape == "credential_name" + else {"model": _CHAT_MODEL, "api_key": _reference(rig.secrets.token), "api_base": rig.provider_base} + ) + seen: Final = len(rig.provider.requests()) + response: Final = rig.owned.gateway.request( + "POST", "/health/test_connection", {"litellm_params": params, "mode": "chat"}, headers=_CLOSE + ) + sent: Final = frozenset(_bearers(rig.provider.requests()[seen:])) + assert response.status_code == 200, response.text + assert JSON_OBJECT.validate_json(response.content)["status"] == "success", response.text + assert sent == {f"Bearer {rig.secrets.current()}"}, sent + + +@pytest.mark.parametrize("case", tuple(_UNREADABLE)) +def test_unreadable_reference_fails_closed_before_any_upstream_call(rig: _Rig, case: str) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, _UNREADABLE[case], marker) + outside_secret: Final = rig.secrets.outside.read_text() + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert outside_secret not in response.text + assert not _anywhere(rig.provider.requests(), outside_secret) + + +def test_unreadable_reference_falls_over_to_the_next_deployment(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-missing-fallback", marker) + sent: Final = frozenset(_bearers(_carrying(rig.provider.requests(), marker))) + assert (response.status_code, sent) == (200, {f"Bearer {rig.secrets.current()}"}), response.text + + +def test_local_token_count_does_not_need_an_unreadable_reference(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/utils/token_counter", + {"model": _UNREADABLE["missing_file"], "messages": [{"role": "user", "content": f"say {marker}"}]}, + headers=_CLOSE, + ) + assert response.status_code == 200, response.text + counted: Final = JSON_OBJECT.validate_json(response.content)["total_tokens"] + assert isinstance(counted, int) and counted > 0, response.text + assert _carrying(rig.provider.requests(), marker) == () + + +def test_input_token_count_with_an_unreadable_reference_fails_closed(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/v1/responses/input_tokens", + {"model": _UNREADABLE["missing_file"], "input": f"say {marker}"}, + headers=_CLOSE, + ) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert str(rig.secrets.allowed) not in response.text, response.text + + +def test_messages_with_an_unreadable_reference_hides_the_token_path(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/v1/messages", + { + "model": _UNREADABLE["missing_file"], + "max_tokens": 16, + "messages": [{"role": "user", "content": f"say {marker}"}], + }, + headers=_CLOSE, + ) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert str(rig.secrets.allowed) not in response.text, response.text + + +def test_unreadable_reference_under_run_secrets_hides_the_token_path(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-run-secrets", marker) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert _RUN_SECRETS_TOKEN not in response.text, response.text + + +@pytest.mark.parametrize("source", ("config_credential", "config_inline")) +def test_provider_token_count_sends_the_current_token(rig: _Rig, source: str) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", "/v1/responses/input_tokens", {"model": rig.model(source), "input": f"say {marker}"}, headers=_CLOSE + ) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.secrets.current()}",)), response.text + assert JSON_OBJECT.validate_json(response.content)["input_tokens"] == _PROVIDER_INPUT_TOKENS, response.text + + +def test_pass_through_with_an_unreadable_reference_hides_the_token_path(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/mistral/v1/chat/completions", + {"model": "mistral-small-latest", "messages": [{"role": "user", "content": f"say {marker}"}]}, + headers=_CLOSE, + ) + assert (response.status_code, _carrying(rig.provider.requests(), marker)) == (401, ()), response.text + assert str(rig.secrets.allowed) not in response.text, response.text + + +def test_admin_configured_endpoint_receives_the_current_token(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-bedrock-endpoint", marker) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.secrets.current()}",)), response.text + + +def _hosts_and_bearers(requests: Sequence[Request]) -> tuple[tuple[str | None, str | None], ...]: + return tuple((request.headers.get("host"), request.headers.get("authorization")) for request in requests) + + +def test_updated_api_base_carries_the_token_to_the_new_host(rig: _Rig) -> None: + first, second = sorted(eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2)) + moved_base: Final = rig.provider_base.replace("127.0.0.1", "localhost") + with rig.owned.gateway.scenario() as scenario: + name, identity = _created_model( + rig.owned.gateway, + scenario, + _deployment( + f"db-moved-{uuid.uuid4().hex}", + {"model": _CHAT_MODEL, "api_key": _reference(rig.secrets.token), "api_base": rig.provider_base}, + ), + ) + eventually(partial(rig.owned.serves_only_on_every_worker, name, identity), bool, seconds=120) + patched: Final = rig.owned.gateway.request( + "PATCH", f"/model/{identity}/update", {"litellm_params": {"api_base": moved_base}} + ) + assert patched.status_code == 200, patched.text + expected: Final = ((200, ((httpx.URL(moved_base).netloc.decode(), f"Bearer {rig.secrets.current()}"),)),) * 2 + reached: Final = eventually( + lambda: tuple( + (status, _hosts_and_bearers(requests)) + for status, requests in (_reached_on_worker(rig, name, idle) for idle in (second, first)) + ), + lambda seen: seen == expected, + seconds=120, + return_last_on_timeout=True, + ) + assert reached == expected, reached + + +def _attack(rig: _Rig, attack: str) -> tuple[str, dict[str, JsonValue]]: + redirected: Final[dict[str, JsonValue]] = {"api_base": rig.intruder_base} + match attack: + case "client_api_base_inline": + return "wif-config-inline", redirected + case "client_api_base_credential": + return "wif-config-credential", redirected + case "body_reference": + return "wif-config-inline", {**redirected, "api_key": _reference(rig.secrets.token)} + case "body_credential_name": + return "wif-config-inline", {**redirected, "litellm_credential_name": "wif-config"} + case "user_config": + injected: Final = {"model": _CHAT_MODEL, "api_key": _reference(rig.secrets.token), **redirected} + return "wif-injected", { + "user_config": {"model_list": [{"model_name": "wif-injected", "litellm_params": injected}]} + } + case _: + pytest.fail(f"unknown attack {attack}") + + +@pytest.mark.parametrize("attack", _ATTACKS) +def test_caller_cannot_send_the_admin_token_to_its_own_host(rig: _Rig, attack: str) -> None: + marker: Final = _marker() + model, extra = _attack(rig, attack) + response: Final = _chat(rig.owned.gateway, model, marker, extra) + intruded: Final = rig.intruder.requests() + assert not _anywhere(intruded, rig.secrets.current()) + assert _carrying(intruded, marker) == (), _bearers(_carrying(intruded, marker)) + assert _carrying(rig.provider.requests(), marker) == () + assert response.status_code in (400, 401), response.text + + +def test_caller_with_its_own_key_still_reaches_its_own_host(rig: _Rig) -> None: + marker: Final = _marker() + own_key: Final = f"sk-caller-{uuid.uuid4().hex}" + response: Final = _chat( + rig.owned.gateway, "wif-config-inline", marker, {"api_base": rig.intruder_base, "api_key": own_key} + ) + intruded: Final = rig.intruder.requests() + assert (response.status_code, _bearers(_carrying(intruded, marker))) == (200, (f"Bearer {own_key}",)), response.text + assert not _anywhere(intruded, rig.secrets.current()) + + +def _team_key(scenario: Scenario, team_id: str, role: str) -> str: + return scenario.key(user_id=scenario.member(team_id, role=role), team_id=team_id) + + +def _team_body(name: str, team_id: str, params: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return _deployment(name, {"model": _CHAT_MODEL, **params}, {"team_id": team_id}) + + +def _team_write_params(rig: _Rig, shape: str) -> dict[str, JsonValue]: + match shape: + case "plain_key": + return {"api_key": f"sk-team-{uuid.uuid4().hex}", "api_base": rig.provider_base} + case "names_credential": + return {"litellm_credential_name": "wif-config"} + case "inline_reference": + return {"api_key": _reference(rig.secrets.token), "api_base": rig.provider_base} + case _: + pytest.fail(f"unknown shape {shape}") + + +def _delete_written(scenario: Scenario, response: httpx.Response) -> None: + if response.status_code == 200: + scenario.delete_model( + string_value(object_value(JSON_OBJECT.validate_json(response.content)["model_info"])["id"]) + ) + + +@pytest.mark.parametrize(("writer", "shape"), tuple(_TEAM_WRITES)) +def test_team_model_write_carrying_a_reference_is_refused(rig: _Rig, writer: str, shape: str) -> None: + with rig.owned.gateway.scenario() as scenario: + team_id: Final = scenario.team() + name: Final = f"wif-team-{uuid.uuid4().hex}" + key: Final = None if writer == "proxy_admin" else _team_key(scenario, team_id, "admin") + response: Final = rig.owned.gateway.request( + "POST", "/model/new", _team_body(name, team_id, _team_write_params(rig, shape)), key=key + ) + stored: Final = len(rig.owned.gateway.rows(_MODEL_ROWS, (name,))) + _delete_written(scenario, response) + expected: Final = _TEAM_WRITES[(writer, shape)] + assert (response.status_code, stored) == (expected, int(expected == 200)), response.text + + +@pytest.mark.parametrize("host", ("credential_host", "other_host")) +def test_team_row_naming_the_credential_sends_the_token_only_to_its_host(rig: _Rig, host: str) -> None: + redirected: Final[dict[str, JsonValue]] = {} if host == "credential_host" else {"api_base": rig.intruder_base} + with rig.owned.gateway.scenario() as scenario: + team_id: Final = scenario.team() + name: Final = f"wif-team-{uuid.uuid4().hex}" + stored_name, identity = _created_model( + rig.owned.gateway, + scenario, + _team_body(name, team_id, {"litellm_credential_name": "wif-config", **redirected}), + ) + eventually(lambda: rig.owned.serves_only_on_every_worker(stored_name, identity), bool, seconds=120) + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, name, marker, key=_team_key(scenario, team_id, "user")) + intruded: Final = rig.intruder.requests() + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert not _anywhere(intruded, rig.secrets.current()) + assert _carrying(intruded, marker) == () + expected: Final = (200, (f"Bearer {rig.secrets.current()}",)) if host == "credential_host" else (401, ()) + assert (response.status_code, sent) == expected, response.text + + +def test_startup_discovery_sends_the_boot_token(rig: _Rig) -> None: + sent: Final = frozenset(_bearers(tuple(request for request in rig.boot if _is_discovery(request)))) + assert sent == {f"Bearer {rig.boot_token}"}, sent + + +def _side_call(rig: _Rig, seam: str, marker: str) -> tuple[httpx.Response, Callable[[Request], bool]]: + gateway: Final = rig.owned.gateway + match seam: + case "realtime_health": + return ( + gateway.request("GET", "/health", params={"model": "wif-realtime"}, headers=_CLOSE), + lambda request: request.target.startswith("/v1/realtime"), + ) + case "vector_store_search": + return ( + gateway.request( + "POST", f"/v1/vector_stores/{_VECTOR_STORE}/search", {"query": f"say {marker}"}, headers=_CLOSE + ), + lambda request: marker in request.body.decode(errors="replace"), + ) + case "batch_probe": + return ( + gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": f"file-{marker}", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": _BATCH_DEPLOYMENT, + }, + headers=_CLOSE, + ), + lambda request: request.target == "/v1/files", + ) + case _: + pytest.fail(f"unknown seam {seam}") + + +_SIDE_STATUS: Final = MappingProxyType({"realtime_health": 503, "vector_store_search": 200, "batch_probe": 400}) + + +@pytest.mark.parametrize("seam", tuple(_SIDE_STATUS)) +def test_calls_outside_the_client_wrapper_send_the_current_token(rig: _Rig, seam: str) -> None: + seen: Final = len(rig.provider.requests()) + response, matches = _side_call(rig, seam, _marker()) + sent: Final = frozenset(_bearers(tuple(request for request in rig.provider.requests()[seen:] if matches(request)))) + assert (response.status_code, sent) == (_SIDE_STATUS[seam], {f"Bearer {rig.secrets.current()}"}), response.text + + +def test_wildcard_listing_shows_the_models_the_provider_lists(rig: _Rig) -> None: + response: Final = rig.owned.gateway.request("GET", "/v1/models", headers=_CLOSE) + assert response.status_code == 200, response.text + entries: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + listed: Final = frozenset(string_value(object_value(entry)["id"]) for entry in entries) + asked: Final = frozenset(_bearers(tuple(request for request in rig.provider.requests() if _is_discovery(request)))) + assert any(_LISTED_MODEL in identity for identity in listed), sorted(listed) + assert not any(bearer.startswith("Bearer oidc/") for bearer in asked if bearer is not None), asked + + +def _stored_rows(rig: _Rig) -> tuple[list[dict[str, JsonValue]], ...]: + return ( + rig.owned.gateway.rows(_CREDENTIAL_ROW, (rig.stored.credential,)), + *(rig.owned.gateway.rows(_MODEL_ROW, (identity,)) for identity in rig.stored.model_ids.values()), + ) + + +def test_resolved_token_is_never_written_back(rig: _Rig) -> None: + before: Final = _stored_rows(rig) + calls: Final = tuple( + _chat(rig.owned.gateway, rig.model(source), _marker()).status_code for source in ("db_credential", "db_inline") + ) + after: Final = _stored_rows(rig) + assert tuple(len(rows) for rows in before) == (1, 1, 1), before + assert after == before, (calls, before, after) diff --git a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py index 91647472f81..5b0f743b836 100644 --- a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py @@ -8,11 +8,14 @@ Tests: 4. strip_advisor_blocks_from_messages with replace_with_text=True """ -from typing import Dict +import sys +from typing import Dict, Final from unittest.mock import AsyncMock, patch import pytest +import litellm + ADVISOR_TOOL = { "type": "advisor_20260301", "name": "advisor", @@ -776,6 +779,19 @@ def test_resolve_advisor_credentials_rejects_api_base_without_api_key(): _resolve_advisor_credentials(tool) +@pytest.mark.parametrize("api_base", [None, "https://other.example"]) +def test_resolve_advisor_credentials_rejects_an_oidc_secret_reference_api_key(api_base): + from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + _resolve_advisor_credentials, + ) + + tool = {**ADVISOR_TOOL, "api_key": "oidc/env/LITELLM_MASTER_KEY", "api_base": api_base} + proxy_settings: Final = {"litellm.proxy.proxy_server": _fake_proxy_server({"allow_client_side_credentials": True})} + with patch.dict(sys.modules, proxy_settings): + with pytest.raises(litellm.BadRequestError, match="oidc/ secret reference"): + _resolve_advisor_credentials(tool) + + def test_resolve_advisor_credentials_validates_api_base_before_use(): from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( _resolve_advisor_credentials, diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 64d057dd006..02be27d921e 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -363,6 +363,227 @@ def test_configuring_a_deployment_still_cannot_carry_federation_fields_inline(ro ) +_SERVICE_ACCOUNT_TOKEN_REFERENCE: Final = "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token" + + +def _user_config_with(litellm_params: dict[str, str]) -> dict[str, object]: + return {"model_list": [{"model_name": "devstral", "litellm_params": {"model": "mistral/devstral-latest", **litellm_params}}]} + + +def _user_config_defaulting_to(litellm_params: dict[str, str]) -> dict[str, object]: + return { + "model_list": [ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_base": "https://attacker.example"}, + } + ], + "default_litellm_params": litellm_params, + } + + +def _user_config_search_tool_with(litellm_params: dict[str, str]) -> dict[str, object]: + return { + "search_tools": [ + { + "search_tool_name": "web", + "litellm_params": {"search_provider": "tavily", "api_base": "https://attacker.example", **litellm_params}, + } + ] + } + + +@pytest.fixture +def oidc_api_key_credential(monkeypatch): + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_values={"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + credential_info={"custom_llm_provider": "mistral"}, + ) + ], + ) + + +def _vector_store_embedding_config_with(embedding_config: dict[str, str]) -> dict[str, object]: + return { + "vector_store_id": "docs", + "custom_llm_provider": "milvus", + "litellm_params": { + "api_base": "https://milvus.example", + "litellm_embedding_config": {"api_base": "https://attacker.example", **embedding_config}, + }, + } + + +def test_a_vector_store_embedding_config_may_carry_a_plain_api_key(): + assert is_request_body_safe( + request_body=_vector_store_embedding_config_with({"api_key": "sk-embedding"}), + general_settings={}, + llm_router=None, + model="", + route="/vector_store/new", + ) + + +@pytest.mark.parametrize("route", [None, "/model/new", "/health/test_connection"]) +@pytest.mark.parametrize( + "body", + [ + {"model": "devstral", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + {"model": "devstral", "extra_body": {"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}}, + {"model": "devstral", "fallbacks": [{"model": "other", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}]}, + { + "model": "devstral", + "user_config": _user_config_with( + {"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE, "api_base": "https://attacker.example"} + ), + }, + {"model": "devstral", "user_config": _user_config_defaulting_to({"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE})}, + {"model": "web", "user_config": _user_config_search_tool_with({"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE})}, + _vector_store_embedding_config_with({"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}), + ], + ids=[ + "top_level", + "extra_body", + "fallback_target", + "user_config", + "user_config_defaults", + "user_config_search_tool", + "vector_store_embedding_config", + ], +) +def test_a_request_body_cannot_carry_an_oidc_api_key_reference(route: str | None, body: dict): + """The call path reads an oidc/ api_key from the proxy's own filesystem or environment, so a body that set one + could send the proxy's service account token to a host the caller picks. No opt-in or route relaxes it.""" + with pytest.raises(ValueError, match="api_key holds an oidc/ secret reference"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + route=route, + ) + + +@pytest.mark.parametrize( + "body", + [ + {"model": "devstral", "litellm_credential_name": "admin-oidc"}, + { + "model": "devstral", + "user_config": _user_config_with( + {"litellm_credential_name": "admin-oidc", "api_base": "https://attacker.example"} + ), + }, + {"model": "devstral", "user_config": _user_config_defaulting_to({"litellm_credential_name": "admin-oidc"})}, + _vector_store_embedding_config_with({"litellm_credential_name": "admin-oidc"}), + ], + ids=["top_level", "user_config", "user_config_defaults", "vector_store_embedding_config"], +) +def test_a_request_body_cannot_name_a_credential_holding_an_oidc_api_key(oidc_api_key_credential, body: dict): + with pytest.raises(ValueError, match=r"names a credential .* oidc/ api_key secret reference"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + ) + + +@pytest.mark.parametrize("route", [None, "/chat/completions"]) +def test_a_call_cannot_carry_an_oidc_api_key_in_litellm_params(route: str | None): + with pytest.raises(ValueError, match="api_key holds an oidc/ secret reference"): + is_request_body_safe( + request_body={"model": "devstral", "litellm_params": {"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + route=route, + ) + + +@pytest.mark.parametrize("route", ["/model/new", "/model/update", "/model/deployment-1/update", "/health/test_connection"]) +def test_a_deployment_write_may_carry_an_oidc_api_key_in_litellm_params(route: str): + """A proxy admin configures an oidc/ api_key on a deployment through these routes, and the model management role + gate refuses anyone else with a 403.""" + assert ( + is_request_body_safe( + request_body={ + "model": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + }, + general_settings={}, + llm_router=None, + model="devstral", + route=route, + ) + is True + ) + + +_OIDC_DEPLOYMENT_PARAMS: Final = ( + pytest.param({"model": "mistral/devstral-latest", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, id="inline"), + pytest.param({"model": "mistral/devstral-latest", "litellm_credential_name": "admin-oidc"}, id="named_credential"), +) + + +@pytest.mark.parametrize("override", ["api_base", "base_url"]) +@pytest.mark.parametrize("deployment_params", _OIDC_DEPLOYMENT_PARAMS) +def test_a_base_override_cannot_send_the_deployments_oidc_api_key( + oidc_api_key_credential, deployment_params: dict, override: str +): + from litellm.exceptions import BadRequestError + from litellm.router_utils.clientside_credential_handler import get_dynamic_litellm_params + + with pytest.raises(BadRequestError, match="api_key is an oidc/ secret reference"): + get_dynamic_litellm_params( + litellm_params=dict(deployment_params), request_kwargs={override: "https://caller.example/v1"} + ) + + +@pytest.mark.parametrize("deployment_params", _OIDC_DEPLOYMENT_PARAMS) +def test_a_base_override_with_the_callers_own_api_key_is_allowed(oidc_api_key_credential, deployment_params: dict): + from litellm.router_utils.clientside_credential_handler import get_dynamic_litellm_params + + out: Final = get_dynamic_litellm_params( + litellm_params=dict(deployment_params), + request_kwargs={"api_base": "https://caller.example/v1", "api_key": "sk-caller"}, + ) + assert (out["api_base"], out["api_key"]) == ("https://caller.example/v1", "sk-caller") + + +def test_test_connection_may_name_a_credential_holding_an_oidc_api_key(oidc_api_key_credential): + """The admin UI tests a stored credential by name, which is how a proxy admin checks an oidc/ api_key works.""" + assert ( + is_request_body_safe( + request_body={"model": "devstral", "litellm_params": {"litellm_credential_name": "admin-oidc"}}, + general_settings={}, + llm_router=None, + model="devstral", + route="/health/test_connection", + ) + is True + ) + + +def test_a_user_config_with_its_own_api_key_is_still_allowed(): + assert ( + is_request_body_safe( + request_body={"model": "devstral", "user_config": _user_config_with({"api_key": "sk-client-owned"})}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + ) + is True + ) + + class TestCustomAuthCommonChecksWarning: """custom_auth_common_checks_warning only warns when custom auth is configured and the common-checks opt-in is off, since that is the only state where diff --git a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py index 00f84e75664..4189a8dba91 100644 --- a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py +++ b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py @@ -3,6 +3,7 @@ import json from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from pathlib import Path from types import MappingProxyType from typing import Final, Literal, cast from unittest.mock import AsyncMock, MagicMock @@ -594,6 +595,38 @@ async def test_litellm_executed_provider_for_leaves_a_server_with_its_own_files_ assert probe.upstreams == [("http://vllm.test/v1/", "vllm-key")] +@pytest.mark.parametrize( + ("model_info", "expected_key"), + [({"id": "vllm-batch"}, "vllm-secret"), ({"id": "vllm-batch", "team_id": "team-1"}, None)], + ids=["proxy-level deployment", "team-scoped deployment"], +) +async def test_resolve_litellm_executed_provider_probes_with_the_secret_only_behind_an_admin_oidc_api_key( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, model_info: Mapping[str, str], expected_key: str | None +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("vllm-secret") + reference: Final = f"oidc/file/{token}" + router: Final = Router( + model_list=[ + { + "model_name": BATCH_MODEL, + "litellm_params": {**VLLM_CREDENTIALS, "api_key": reference}, + "model_info": dict(model_info), + } + ] + ) + probe: Final = FakeFilesApiProbe(True, []) + + try: + provider: Final = await resolve_litellm_executed_provider(router, "vllm-batch", "team-1", probe) + finally: + router.discard() + + assert provider == "hosted_vllm" + assert probe.upstreams == [("http://vllm.test/v1/", expected_key or reference)] + + @pytest.mark.parametrize( "credentials", [{"custom_llm_provider": "openai", "model": "gpt-4o", "api_base": "http://openai.test/v1"}, {"model": 7}], diff --git a/tests/unit/proxy/common_utils/test_credential_hydration.py b/tests/unit/proxy/common_utils/test_credential_hydration.py index f40b0114d71..ad75b5b8760 100644 --- a/tests/unit/proxy/common_utils/test_credential_hydration.py +++ b/tests/unit/proxy/common_utils/test_credential_hydration.py @@ -1,10 +1,16 @@ +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest import litellm -from litellm.proxy.common_utils.credential_hydration import hydrate_named_credential_authoritative +from litellm.proxy.common_utils.credential_hydration import ( + effective_server_owned_wif_fields, + hydrate_named_credential_authoritative, + named_credential_wif_fields, +) from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.types.utils import CredentialItem @pytest.mark.asyncio @@ -26,3 +32,46 @@ async def test_authoritative_hydrate_returns_an_encrypted_empty_value_as_empty(m assert resolved is not None assert resolved.credential_values == {"api_base": "", "openai_service_account_id": "user-1"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "credential_api_key, expected", + [ + ("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", ("api_key",)), + ("sk-static", ()), + ], + ids=["oidc_reference", "static_key"], +) +async def test_attaching_a_credential_is_server_owned_only_when_its_api_key_is_an_oidc_reference( + credential_api_key: str, expected: tuple[str, ...] +): + credential: Final = CredentialItem( + credential_name="admin-oidc", credential_values={"api_key": credential_api_key}, credential_info={} + ) + + with patch.object(litellm, "credential_list", [credential]): # test-quality-ok: the in-memory list is the input + fields = await effective_server_owned_wif_fields( + {"model": "mistral/devstral-latest"}, {"litellm_credential_name": "admin-oidc"}, None + ) + + assert fields == expected + + +@pytest.mark.asyncio +async def test_a_stored_credential_whose_encrypted_api_key_is_an_oidc_reference_is_server_owned(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-hydration-test-salt") + row = { + "credential_name": "admin-oidc", + "credential_values": { + "api_key": encrypt_value_helper("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token") + }, + "credential_info": {"custom_llm_provider": "mistral"}, + } + prisma = MagicMock() + prisma.db.litellm_credentialstable.find_unique = AsyncMock(return_value=row) + + with patch.object(litellm, "credential_list", []): # test-quality-ok: a pod that has not loaded the row yet + fields = await named_credential_wif_fields("admin-oidc", prisma) + + assert fields == ("api_key",) diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 4359ea6592b..6aeceaf4356 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -17,6 +17,7 @@ import litellm from litellm.models.credentials import CredentialSource from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.credential_endpoints.endpoints import get_llm_router from litellm.proxy.proxy_server import app from litellm.types.utils import CredentialItem @@ -659,6 +660,29 @@ class TestNonAdminCannotPersistWifFieldsOnCredential: assert response.status_code == 403, response.text + @pytest.mark.parametrize( + "auth, expected_status", + [(_as_non_admin, 403), (_as_admin, 200)], + ids=["non_admin", "proxy_admin"], + ) + def test_only_a_proxy_admin_can_create_a_credential_with_an_oidc_api_key( + self, restore_credential_list, auth, expected_status: int + ): + """The call path reads an oidc/ api_key from the proxy's own filesystem, so it is a server-owned value the + same way a federation field is.""" + with _repository_holding(None) as repository: + response = _post_credential( + { + "credential_name": "mistral-sa", + "credential_values": {"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + "credential_info": {"custom_llm_provider": "mistral"}, + }, + auth=auth, + ) + + assert response.status_code == expected_status, response.text + assert repository.create.await_count == (1 if expected_status == 200 else 0) + def test_non_admin_can_create_a_credential_without_wif_fields(self, restore_credential_list): with _repository_holding(None) as repository: response = _post_credential( @@ -921,6 +945,32 @@ class TestNonAdminCannotTouchAStoredWifCredential: assert "anthropic_keycloak_token_url" in response.text repository.update_by_name.assert_not_awaited() + def test_non_admin_cannot_repoint_a_stored_credential_whose_encrypted_api_key_is_an_oidc_reference( + self, restore_credential_list, monkeypatch + ): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-credential-endpoints-test-salt") + stored = CredentialItem( + credential_name="mistral-sa", + credential_values={ + "api_key": encrypt_value_helper("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token") + }, + credential_info={"custom_llm_provider": "mistral"}, + ) + with _repository_holding(stored) as repository: + response = _patch_credential( + "mistral-sa", + { + "credential_name": "mistral-sa", + "credential_values": {"api_base": "https://attacker.example"}, + "credential_info": {}, + }, + auth=_as_non_admin, + ) + + assert response.status_code == 403, response.text + assert "'api_key'" in response.text + repository.update_by_name.assert_not_awaited() + def test_proxy_admin_can_delete_a_wif_field_off_a_credential(self, restore_credential_list): with _repository_holding(_wif_credential("federated-cred")) as repository: response = _patch_credential( diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 441c52b6a5c..d5288a7fc12 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -6,6 +6,7 @@ from collections.abc import Awaitable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime, timedelta from importlib import import_module +from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -740,6 +741,73 @@ def _test_connection_probe( yield ahealth_check +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_api_key", "request_params"), + [ + ("oidc/file/{token}", {"model": "mistral/devstral-latest"}), + ("sk-stale", {"model": "mistral/devstral-latest", "api_key": "oidc/file/{token}"}), + ], + ids=["stored on the deployment", "typed into the probe by an admin"], +) +async def test_test_model_connection_sends_the_token_behind_an_admin_oidc_api_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + deployment_api_key: str, + request_params: Mapping[str, str], +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = tmp_path / "token" + token.write_text("mistral-sa-token") + router: Final = Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": { + "model": "mistral/devstral-latest", + "api_key": deployment_api_key.format(token=token), + }, + "model_info": {"id": "devstral-id"}, + } + ] + ) + + try: + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.premium_user", False), + respx.mock(assert_all_called=True) as upstream, + ): + upstream.post("https://api.mistral.ai/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-probe", + "object": "chat.completion", + "created": 1, + "model": "devstral-latest", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + result: Final = await health_test_model_connection( + request=MagicMock(), + mode="chat", + litellm_params={key: value.format(token=token) for key, value in request_params.items()}, + model_info={"id": "devstral-id"}, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"), + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + finally: + router.discard() + + assert result["status"] == "success" + assert sent == "Bearer mistral-sa-token" + + MANTLE_CLAUDE_DEPLOYMENT: Final = MappingProxyType( { "model_name": "claude-haiku-4-5", diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index bc714ad1cbc..303b8e060ed 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -518,6 +518,43 @@ class TestModelManagementAuthChecks: ) assert result is True + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("model_info", "api_key", "refused"), + [ + ({"team_id": "test_team"}, "oidc/env/MISTRAL_SA_TOKEN", True), + ({}, "oidc/env/MISTRAL_SA_TOKEN", False), + ({"team_id": "test_team"}, "sk-static", False), + ], + ids=["team model with a reference", "proxy-level model with a reference", "team model with a plain key"], + ) + async def test_an_admin_cannot_save_an_oidc_api_key_reference_on_a_team_model( + self, model_info: dict, api_key: str, refused: bool + ): + """A team model's inline api_key is never resolved, so saving an oidc/ reference there would only ever send + the reference text upstream.""" + model_params = Deployment( + model_name="test_model", + litellm_params=LiteLLM_Params(model="mistral/devstral-latest", api_key=api_key), + model_info=model_info, + ) + + async def save() -> object: + return await ModelManagementAuthChecks.can_user_make_model_call( + model_params=model_params, + user_api_key_dict=self.admin_user, + prisma_client=MockPrismaClient(team_exists=True), + premium_user=True, + incoming_params={"model": "mistral/devstral-latest", "api_key": api_key}, + ) + + if not refused: + assert await save() is True + return + with pytest.raises(Exception, match="team model cannot hold an oidc/ api_key secret reference") as exc_info: + await save() + assert (getattr(exc_info.value, "param", None), getattr(exc_info.value, "code", None)) == ("api_key", "400") + @pytest.mark.asyncio async def test_can_user_make_model_call_normal_user_fails(self): """Test that normal users cannot make model calls""" @@ -8838,6 +8875,63 @@ class TestNonAdminCannotPersistWifFieldsOnModel: authorized for a team-scoped model must not be able to set one via /model/new, /model/update, or PATCH /model/{id}/update; a proxy admin still can.""" + @pytest.mark.asyncio + @pytest.mark.parametrize( + "stored_api_key, patched_params", + [ + ("sk-static", {"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}), + ("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", {"api_base": "https://attacker.example"}), + ], + ids=["sets_the_reference", "redirects_a_stored_reference"], + ) + async def test_patch_model_non_admin_cannot_touch_an_oidc_api_key_reference( + self, stored_api_key: str, patched_params: dict + ): + """The call path reads an oidc/ api_key from the proxy's own filesystem, so a team admin who could set one, or + move a deployment that has one to another api_base, could send the proxy's token to a host they pick.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + patch_model, + ) + + non_admin = UserAPIKeyAuth(user_id="team_admin", user_role=LitellmUserRoles.INTERNAL_USER) + existing_row = MagicMock() + existing_row.litellm_params = {"model": "mistral/devstral-latest", "api_key": stored_api_key} + existing_row.model_dump.return_value = { + "model_name": "devstral", + "litellm_params": existing_row.litellm_params, + "model_info": {"id": "m1"}, + } + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + + with ( + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", + mock_prisma, + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.llm_router", + MagicMock(**{"get_model_ids.return_value": ["m1"]}), + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.store_model_in_db", + True, + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.premium_user", + True, + ), + ): + with pytest.raises(Exception, match="oidc/ api_key secret reference") as exc_info: + await patch_model( + model_id="m1", + patch_data=updateDeployment(litellm_params=updateLiteLLMParams(**patched_params)), + user_api_key_dict=non_admin, + ) + assert getattr(exc_info.value, "param", "") == "api_key" + mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() + @pytest.mark.asyncio async def test_patch_model_non_admin_cannot_set_wif_field(self): from litellm.proxy.management_endpoints.model_management_endpoints import ( diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 941accff314..9c50a6d4df6 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -449,19 +449,21 @@ def test_oidc_file_reference_outside_allowed_dirs_is_refused(tmp_path, monkeypat _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) assert excinfo.value.status_code == 401 - assert f"oidc/file/{outside}" in str(excinfo.value.detail) + assert str(outside) not in str(excinfo.value.detail) assert "outside the allowed credential directories" in str(excinfo.value.detail) -def test_missing_oidc_token_file_is_a_clear_auth_error(tmp_path, monkeypatch): - monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) - monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{tmp_path / 'missing-token'}") +def test_missing_oidc_token_file_is_a_clear_auth_error(monkeypatch): + monkeypatch.delenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "oidc/file//run/secrets/lit-missing-token") with pytest.raises(HTTPException) as excinfo: _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) assert excinfo.value.status_code == 401 - assert f"oidc/file/{tmp_path / 'missing-token'}" in str(excinfo.value.detail) + assert "pass-through credential oidc/file" in str(excinfo.value.detail) + assert "No such file or directory" in str(excinfo.value.detail) + assert "/run/secrets" not in str(excinfo.value.detail) def test_oidc_env_reference_is_resolved(monkeypatch): diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 2943fe79979..4647952312a 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -7181,6 +7181,76 @@ async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_ assert value not in logged +_SERVICE_ACCOUNT_TOKEN_REFERENCE: Final = "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token" + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_refuses_a_forwarded_x_api_key_holding_an_oidc_reference(): + """A forwarded provider key lands in api_key after auth, so the body check cannot see it and a client could get + the proxy to read its own service account token for the call.""" + from fastapi import HTTPException + + request_mock = _make_request_mock( + "/v1/messages", + { + "Content-Type": "application/json", + "x-litellm-api-key": "Bearer sk-virtual-key", + "x-api-key": _SERVICE_ACCOUNT_TOKEN_REFERENCE, + }, + ) + + with pytest.raises(HTTPException) as exc_info: + await add_litellm_data_to_request( + data={"model": "claude-sonnet-5", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"forward_llm_provider_auth_headers": True}, + version="test-version", + ) + + assert exc_info.value.status_code == 400 + assert "api_key holds an oidc/ secret reference" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_refuses_a_model_config_credential_holding_an_oidc_reference(monkeypatch): + """Team and project metadata are written by non-admins, so a model_config naming a credential whose api_key is an + oidc/ reference must not get that reference resolved for a deployment the team controls.""" + from fastapi import HTTPException + + monkeypatch.setattr(litellm, "enable_model_config_credential_overrides", True) + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_info={}, + credential_values={"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + ) + ], + ) + team_key: Final = UserAPIKeyAuth( + api_key="hashed-key", + team_id="team-1", + team_metadata={"model_config": {"azure/gpt-4o": {"azure": {"litellm_credentials": "admin-oidc"}}}}, + ) + + with pytest.raises(HTTPException) as exc_info: + await add_litellm_data_to_request( + data={"model": "azure/gpt-4o", "messages": [{"role": "user", "content": "hello"}]}, + request=_make_request_mock("/v1/chat/completions", {"Content-Type": "application/json"}), + user_api_key_dict=team_key, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert exc_info.value.status_code == 400 + assert "api_key holds an oidc/ secret reference" in str(exc_info.value.detail) + + @pytest.mark.parametrize( "header, expected_redacted", [ diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 18fe5a2ad32..5cbda9d320c 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -199,6 +199,76 @@ async def test_gpt_token_counting(): assert response.request_model == "gpt-4" + +@pytest.mark.asyncio +async def test_token_counter_leaves_the_routers_deployment_without_the_credentials_it_resolves(tmp_path, monkeypatch): + """token_counter reads the router's own deployment, so loading the named credential into it would leave the + token read from the file on the shared deployment for every later request.""" + from litellm.types.utils import CredentialItem + + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token_file = tmp_path / "token" + token_file.write_text("sa-token-from-file") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="mistral-sa", + credential_values={"api_key": f"oidc/file/{token_file}"}, + credential_info={}, + ) + ], + ) + llm_router = Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "litellm_credential_name": "mistral-sa"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + + response = await token_counter( + request=TokenCountRequest(model="devstral", messages=[{"role": "user", "content": "hello"}]) + ) + + assert response.total_tokens > 0 + stored = (await llm_router.async_get_available_deployment(model="devstral", request_kwargs={}))["litellm_params"] + assert "api_key" not in stored + assert stored["litellm_credential_name"] == "mistral-sa" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_endpoint", [False, True], ids=["local tokenizer", "provider count api"]) +async def test_token_counter_reads_an_oidc_api_key_only_when_it_calls_the_provider( + tmp_path, monkeypatch, call_endpoint +): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + llm_router = Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_key": f"oidc/file/{tmp_path / 'missing-token'}", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + request = TokenCountRequest(model="claude", messages=[{"role": "user", "content": "hello"}]) + + if not call_endpoint: + assert (await token_counter(request=request, call_endpoint=call_endpoint)).total_tokens > 0 + return + with pytest.raises(ProxyException) as excinfo: + await token_counter(request=request, call_endpoint=call_endpoint) + assert excinfo.value.code == "401" + assert str(tmp_path) not in excinfo.value.message + + @pytest.mark.asyncio async def test_anthropic_messages_count_tokens_endpoint(): """ diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 527d3d599b1..4d89ee497a5 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -527,6 +527,29 @@ async def test_bearer_health_check_sends_stored_credential_as_bearer_token( assert connect.url == expected_url +@pytest.mark.asyncio +async def test_bearer_health_check_sends_the_secret_behind_an_oidc_credential( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("sk-from-token-file") + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="voice-sa", credential_values={"api_key": f"oidc/file/{token}"}, credential_info={})], + ) + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime", + custom_llm_provider="openai", + api_key=None, + model_params={"model": "openai/gpt-realtime", "litellm_credential_name": "voice-sa"}, + ) + assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer sk-from-token-file"} + + @pytest.mark.asyncio async def test_azure_health_check_probes_ga_transcription_url_for_transcription_model(local_model_cost_map): """Regression for LIT-6240: transcription-only models (mode audio_transcription diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index e11c5d6c2f3..f956bb66e6c 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -5511,6 +5511,48 @@ class TestRouterPreRoutingSharedAliasName: assert sent["aws_region_name"] == "us-west-2" assert sent["drop_params"] is True + @pytest.mark.asyncio + @pytest.mark.parametrize( + "tier_params", + [ + { + "api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", + "api_base": "https://attacker.example", + }, + {"litellm_credential_name": "admin-oidc", "api_base": "https://attacker.example"}, + ], + ids=["inline_reference", "credential_holding_a_reference"], + ) + async def test_a_tier_cannot_make_the_proxy_read_an_oidc_api_key(self, monkeypatch, tier_params: dict): + """A team admin can write an auto-router's tiers, and tier params beat the target deployment's own, so a + tier could otherwise send the proxy's own token file to a host the tier picks.""" + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_values={"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + credential_info={}, + ) + ], + ) + marker = self._region_marker_entry() + marker["litellm_params"]["complexity_router_config"] = { + "tiers": { + tier: {"model_name": "bedrock-tier", "litellm_params": tier_params} + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + } + } + tier_target = self._bedrock_tier_entry() + tier_target["litellm_params"]["mock_response"] = "sent upstream" + router = Router(model_list=[marker, tier_target]) + + with pytest.raises(ValueError, match="oidc/"): + await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) + @pytest.mark.asyncio async def test_a_markers_explicit_flag_beats_the_tiers_pydantic_default(self): """Every deployment materializes `LiteLLM_Params` defaults such as diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 383a1e25e98..09ce002353f 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -1265,6 +1265,43 @@ async def test_a_stored_fallback_target_cannot_carry_an_openai_federation_field( ) +@pytest.mark.parametrize( + "target", + [ + {"model": "mistral-backup", "api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + {"model": "mistral-backup", "litellm_credential_name": "admin-oidc", "api_base": "https://attacker.example"}, + ], + ids=["inline_reference", "credential_holding_a_reference"], +) +@pytest.mark.asyncio +async def test_a_stored_fallback_target_cannot_make_the_proxy_read_an_oidc_api_key(monkeypatch, target: dict): + """A stored key or team fallback is merged into kwargs ahead of the call, so it could otherwise point the + proxy's own token file at a host the fallback picks.""" + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_values={"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + credential_info={"custom_llm_provider": "mistral"}, + ) + ], + ) + + with pytest.raises(ValueError, match="oidc/"): + await run_async_fallback( + litellm_router=FakeRouter(), + fallback_model_group=[target], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + ) + + @pytest.mark.asyncio async def test_the_refusal_is_not_swallowed_as_a_fallback_error(): """Checked before the per-target loop on purpose: inside it, the refusal would be caught as diff --git a/tests/unit/secret_managers/test_secret_managers_main.py b/tests/unit/secret_managers/test_secret_managers_main.py index d32555c3669..7c6e1de6949 100644 --- a/tests/unit/secret_managers/test_secret_managers_main.py +++ b/tests/unit/secret_managers/test_secret_managers_main.py @@ -9,10 +9,12 @@ from unittest.mock import Mock, patch import pytest import litellm +from litellm._logging import redact_internal_details_from_client_message from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.secret_managers.main import ( get_secret, normalize_nonempty_secret_str, + resolve_local_oidc_reference, secret_manager_would_be_consulted, ) from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem @@ -322,6 +324,15 @@ def test_oidc_file_success(tmp_path, monkeypatch): assert result == "file_token" +def test_oidc_file_reads_a_token_under_a_directory_named_oidc(tmp_path, monkeypatch): + token_dir: Final = tmp_path / "oidc" + token_dir.mkdir() + (token_dir / "token").write_text("nested_token") + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + + assert get_secret(f"oidc/file/{token_dir}/token") == "nested_token" + + def test_oidc_file_rejects_path_outside_allowlist(tmp_path, monkeypatch): outside_file = tmp_path / "outside.txt" outside_file.write_text("should_not_read") @@ -334,10 +345,12 @@ def test_oidc_file_rejects_path_outside_allowlist(tmp_path, monkeypatch): get_secret(f"oidc/file/{outside_file}") -def test_oidc_file_rejects_relative_path(tmp_path, monkeypatch): +def test_oidc_file_rejects_relative_path_with_a_hint_client_redaction_leaves_whole(tmp_path, monkeypatch): monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) - with pytest.raises(ValueError, match="must be absolute"): + with pytest.raises(ValueError, match="must be absolute") as excinfo: get_secret("oidc/file/relative/path/token") + hint: Final = str(excinfo.value) + assert redact_internal_details_from_client_message(hint) == hint def test_oidc_env_success(mock_env): @@ -474,3 +487,15 @@ def test_get_secret_turns_only_boolean_literals_from_the_secret_manager_into_boo assert secret == expected assert type(secret) is type(expected) + + +def test_an_unresolvable_reference_cannot_forge_a_server_log_line( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.delenv("LIT_UNSET_SA_TOKEN", raising=False) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + resolve_local_oidc_reference("oidc/env/LIT_UNSET_SA_TOKEN\nFORGED ENTRY") + + assert "LIT_UNSET_SA_TOKEN" in caplog.text + assert "\nFORGED" not in caplog.text diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 094a13c37b2..4cce5b50f0b 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -18,6 +18,7 @@ import queue import re import sys import threading +from pathlib import Path from types import MappingProxyType from typing import Final, cast from unittest.mock import Mock, patch @@ -172,6 +173,35 @@ async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch _invalidate_model_cost_lowercase_map() +async def test_discovery_sends_the_secret_behind_an_oidc_api_key(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("local-secret") + router: Final = Router(model_list=[{ + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://discovery.test/v1", + "api_key": f"oidc/file/{token}", + }, + "model_info": {"id": "oidc-deployment"}, + }]) + authorizations: Final[list[str | None]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + authorizations.append(request.headers.get("Authorization")) + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await router._arefresh_deployment_model_info(router.model_list[0], client=handler) + assert authorizations == ["Bearer local-secret"] + _invalidate_model_cost_lowercase_map() + + async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) first, second = tuple( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6ffb589fd41..7df4910c24f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -59,6 +59,7 @@ from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_K from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams, LiteLLM_Params from litellm.types.utils import ( + CredentialItem, ADDRESSED_RESPONSE_ID_FIELD, CallTypes, Choices, @@ -7004,6 +7005,555 @@ def test_load_credentials_from_list_fills_kwargs_from_the_loaded_credential_with assert _credential_warnings(caplog) == [] +_MISTRAL_CHAT_URL: Final = "https://api.mistral.ai/v1/chat/completions" +_MISTRAL_REPLY: Final = { + "id": "chatcmpl-oidc", + "object": "chat.completion", + "created": 0, + "model": "devstral-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "pong"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _allowed_token_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, content: str) -> Path: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text(content) + return token + + +def _mistral_credential(api_key: str) -> CredentialItem: + return CredentialItem(credential_name="mistral-sa", credential_values={"api_key": api_key}, credential_info={}) + + +_PING: Final = [{"role": "user", "content": "ping"}] + + +def _ping_mistral_through(credential_name: str) -> None: + litellm.completion(model="mistral/devstral-latest", messages=_PING, litellm_credential_name=credential_name) + + +def test_named_credential_oidc_file_api_key_sends_the_token_read_on_each_call( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "token-before-rotation") + reference: Final = f"oidc/file/{token}" + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(reference)]) + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + _ping_mistral_through("mistral-sa") + token.write_text("token-after-rotation") + _ping_mistral_through("mistral-sa") + sent: Final = [call.request.headers["Authorization"] for call in upstream.calls] + + assert sent == ["Bearer token-before-rotation", "Bearer token-after-rotation"] + assert litellm.credential_list[0].credential_values["api_key"] == reference + + +@pytest.mark.asyncio +async def test_deployment_oidc_env_api_key_resolves_per_call_and_the_router_keeps_the_reference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token-one") + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": "oidc/env/MISTRAL_SA_TOKEN"}, + } + ] + ) + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + await router.acompletion(model="devstral", messages=[{"role": "user", "content": "ping"}]) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token-two") + await router.acompletion(model="devstral", messages=[{"role": "user", "content": "ping"}]) + sent: Final = [call.request.headers["Authorization"] for call in upstream.calls] + + assert sent == ["Bearer env-token-one", "Bearer env-token-two"] + assert router.get_model_list(model_name="devstral")[0]["litellm_params"]["api_key"] == "oidc/env/MISTRAL_SA_TOKEN" + + +def test_explicit_api_key_wins_over_a_named_credential_reference( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(f"oidc/file/{tmp_path / 'absent'}")]) + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + litellm.completion( + model="mistral/devstral-latest", + messages=_PING, + litellm_credential_name="mistral-sa", + api_key="sk-explicit", + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + + assert sent == "Bearer sk-explicit" + + +@pytest.mark.parametrize( + ("make_reference", "reason"), + [ + (lambda tmp_path: f"oidc/file/{tmp_path / 'absent'}", "No such file or directory"), + (lambda tmp_path: f"oidc/file/{tmp_path.parent}/outside", "outside the allowed credential directories"), + (lambda tmp_path: "oidc/file//run/secrets/lit-absent-token", "outside the allowed credential directories"), + (lambda tmp_path: "oidc/file/relative/token", "must be absolute"), + (lambda tmp_path: "oidc/env/LIT_UNSET_SA_TOKEN", "environment variable is not set"), + (lambda tmp_path: f"oidc/file/{tmp_path / 'empty'}", "empty value"), + (lambda tmp_path: f"oidc/file/{tmp_path / 'binary'}", "not UTF-8 text"), + ], +) +def test_unresolvable_oidc_api_key_fails_closed_naming_the_reason_without_its_target_or_content( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + make_reference: Callable[[Path], str], + reason: str, +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.delenv("LIT_UNSET_SA_TOKEN", raising=False) + (tmp_path / "empty").write_text("") + (tmp_path / "binary").write_bytes(b"\xffFILE-CONTENT") + (tmp_path.parent / "outside").write_text("FILE-CONTENT") + reference: Final = make_reference(tmp_path) + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(reference)]) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(litellm.AuthenticationError) as excinfo: + _ping_mistral_through("mistral-sa") + + assert not route.called + assert excinfo.value.status_code == 401 + assert reason in excinfo.value.message + assert reference.split("/", 2)[2] not in excinfo.value.message + assert reference in caplog.text + assert "FILE-CONTENT" not in excinfo.value.message + assert "0xff" not in excinfo.value.message + + +@pytest.mark.parametrize( + ("reference", "kind"), + [ + ("oidc/env_path/MISTRAL_SA_TOKEN_PATH", "oidc/env_path"), + ("oidc/google/https://api.mistral.ai", "oidc/google"), + ("oidc/github/mistral", "oidc/github"), + ("oidc/env_MISTRAL_SA_TOKEN_PATH", "oidc/"), + ("oidc/LIT-UNKNOWN-KIND/MISTRAL_SA_TOKEN_PATH", "oidc/"), + ], +) +def test_oidc_api_key_of_a_kind_api_key_cannot_resolve_fails_closed_as_an_authentication_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, reference: str, kind: str +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "must-not-be-sent") + monkeypatch.setenv("MISTRAL_SA_TOKEN_PATH", str(token)) + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(reference)]) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + with pytest.raises(litellm.AuthenticationError) as excinfo: + _ping_mistral_through("mistral-sa") + + assert not route.called + assert excinfo.value.status_code == 401 + assert f"{kind} secret reference" in excinfo.value.message + assert reference.removeprefix(kind).lstrip("/") not in excinfo.value.message + assert "MISTRAL_SA_TOKEN_PATH" not in excinfo.value.message + + +@pytest.mark.asyncio +async def test_router_fails_over_from_a_deployment_whose_oidc_api_key_kind_cannot_resolve( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": { + "model": "mistral/devstral-latest", + "api_key": "oidc/google/https://api.mistral.ai", + "order": 1, + }, + }, + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": "sk-healthy", "order": 2}, + }, + ], + num_retries=1, + retry_after=0, + ) + + try: + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + response: Final = await router.acompletion(model="devstral", messages=_PING) + sent: Final = [call.request.headers["Authorization"] for call in upstream.calls] + finally: + router.discard() + + assert response.choices[0].message.content == "pong" + assert sent == ["Bearer sk-healthy"] + + +def test_an_oidc_api_key_sent_with_the_call_goes_upstream_as_written( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "server-secret") + reference: Final = f"oidc/file/{token}" + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + litellm.completion(model="mistral/devstral-latest", messages=_PING, api_key=reference) + sent: Final = upstream.calls.last.request.headers["Authorization"] + + assert sent == f"Bearer {reference}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "expected"), + [({}, "Bearer env-token"), ({"team_id": "team-1"}, "Bearer oidc/env/MISTRAL_SA_TOKEN")], + ids=["proxy-level deployment", "team-scoped deployment"], +) +async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( + monkeypatch: pytest.MonkeyPatch, model_info: Mapping[str, str], expected: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": "oidc/env/MISTRAL_SA_TOKEN"}, + "model_info": dict(model_info), + } + ] + ) + + try: + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + await router.acompletion( + model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + finally: + router.discard() + + assert sent == expected + + +_DEPLOYMENT_HOST_REFERENCE_REFUSAL: Final = "goes only to the host its deployment configures" +_BEDROCK_RUNTIME: Final = "https://bedrock-runtime.us-east-1.amazonaws.com" +_BEDROCK_CLAUDE: Final = "us.anthropic.claude-sonnet-5-5" +_BEDROCK_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "pong"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, +} + + +def _bedrock_router(api_key: str, endpoint: str | None) -> litellm.Router: + params: Final = {"model": f"bedrock/converse/{_BEDROCK_CLAUDE}", "aws_region_name": "us-east-1", "api_key": api_key} + return litellm.Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {**params, "aws_bedrock_runtime_endpoint": endpoint} if endpoint else params, + } + ], + num_retries=0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_endpoint", "call_endpoint"), + [(None, "https://caller.example"), (_BEDROCK_RUNTIME, f"{_BEDROCK_RUNTIME}:8443")], + ids=["deployment without an endpoint", "endpoint on another port"], +) +async def test_an_inline_oidc_api_key_is_refused_before_its_file_is_read_when_the_call_picks_another_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, deployment_endpoint: str | None, call_endpoint: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + router: Final = _bedrock_router("oidc/file//run/secrets/lit-missing-token", deployment_endpoint) + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_BEDROCK_REPLY) + with pytest.raises(litellm.AuthenticationError, match=_DEPLOYMENT_HOST_REFERENCE_REFUSAL) as excinfo: + await router.acompletion(model="claude", messages=_PING, aws_bedrock_runtime_endpoint=call_endpoint) + finally: + router.discard() + + assert upstream.calls.call_count == 0 + assert "/run/secrets" not in excinfo.value.message + + +@pytest.mark.asyncio +async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_configures_when_the_call_names_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = _allowed_token_file(tmp_path, monkeypatch, "file-token") + router: Final = _bedrock_router(f"oidc/file/{token}", _BEDROCK_RUNTIME) + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_BEDROCK_REPLY) + await router.acompletion( + model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" + ) + sent: Final = tuple( + (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls + ) + finally: + router.discard() + + assert sent == (("bedrock-runtime.us-east-1.amazonaws.com", "Bearer file-token"),) + + +_HUGGINGFACE_EMBEDDING: Final = [[0.1, 0.2]] + + +def _authorizations(calls: respx.models.CallList) -> tuple[tuple[str, str], ...]: + return tuple( + (call.request.url.host, call.request.headers["Authorization"]) + for call in calls + if "Authorization" in call.request.headers + ) + + +def _huggingface_router(model: str, api_key: str) -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": model, "litellm_params": {"model": model, "api_key": api_key}}], + num_retries=0, + ) + + +@pytest.mark.asyncio +async def test_an_inline_oidc_api_key_is_refused_when_a_wildcard_call_names_its_own_url_as_the_model( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = _allowed_token_file(tmp_path, monkeypatch, "file-token") + router: Final = _huggingface_router("huggingface/*", f"oidc/file/{token}") + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_HUGGINGFACE_EMBEDDING) + with pytest.raises(litellm.AuthenticationError, match=_DEPLOYMENT_HOST_REFERENCE_REFUSAL): + await router.aembedding( + model="huggingface/https://caller.example/embed", input=["hello"], input_type="feature-extraction" + ) + finally: + router.discard() + + assert _authorizations(upstream.calls) == () + + +@pytest.mark.asyncio +async def test_an_inline_oidc_api_key_reaches_the_url_its_deployment_names_as_the_model( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = _allowed_token_file(tmp_path, monkeypatch, "file-token") + model: Final = "huggingface/https://endpoint.example/embed" + router: Final = _huggingface_router(model, f"oidc/file/{token}") + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_HUGGINGFACE_EMBEDDING) + await router.aembedding(model=model, input=["hello"], input_type="feature-extraction") + sent: Final = _authorizations(upstream.calls) + finally: + router.discard() + + assert sent == (("endpoint.example", "Bearer file-token"),) + + +_CALLER_HOST_REFERENCE_REFUSAL: Final = "sends it only to that credential's own api_base" + + +@pytest.mark.parametrize( + ("credential_api_base", "call_api_base"), + [ + ("https://api.mistral.ai/v1", "https://caller.example/v1"), + (None, "https://caller.example/v1"), + ("https://api.mistral.ai/v1", "http://api.mistral.ai/v1"), + ("https://api.mistral.ai/v1", "https://api.mistral.ai@caller.example/v1"), + ("https://api.mistral.ai/v1", "https://api.mistral.ai:8443/v1"), + ("https://api.mistral.ai/v1", "http://[api.mistral.ai/v1"), + ("https://mistral-gateway.example/v1", None), + ], + ids=[ + "another host", + "credential without api_base", + "another scheme", + "userinfo trick", + "another port", + "unparseable", + "call clears the api_base", + ], +) +def test_a_named_credentials_oidc_api_key_never_goes_to_an_api_base_the_credential_does_not_set( + monkeypatch: pytest.MonkeyPatch, credential_api_base: str | None, call_api_base: str | None +) -> None: + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + credential_values: Final = {"api_key": "oidc/env/MISTRAL_SA_TOKEN", "api_base": credential_api_base} + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="mistral-sa", credential_values=credential_values, credential_info={})], + ) + + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_MISTRAL_REPLY) + with pytest.raises(litellm.AuthenticationError, match=_CALLER_HOST_REFERENCE_REFUSAL): + litellm.completion( + model="mistral/devstral-latest", + messages=_PING, + litellm_credential_name="mistral-sa", + api_base=call_api_base, + ) + + assert upstream.calls.call_count == 0 + + +@pytest.mark.parametrize( + ("credential_api_base", "call_api_base"), + [ + ("https://api.mistral.ai/v1", "https://api.mistral.ai:443/v1"), + ("https://api.mistral.ai:443/v1", "https://API.mistral.ai./v1"), + ], + ids=["explicit default port on the call", "explicit default port on the credential"], +) +def test_a_named_credentials_oidc_api_key_reaches_an_api_base_on_the_same_origin( + monkeypatch: pytest.MonkeyPatch, credential_api_base: str, call_api_base: str +) -> None: + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + credential_values: Final = {"api_key": "oidc/env/MISTRAL_SA_TOKEN", "api_base": credential_api_base} + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="mistral-sa", credential_values=credential_values, credential_info={})], + ) + + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_MISTRAL_REPLY) + litellm.completion( + model="mistral/devstral-latest", + messages=_PING, + litellm_credential_name="mistral-sa", + api_base=call_api_base, + ) + sent: Final = tuple(authorization for _, authorization in _authorizations(upstream.calls)) + + assert sent == ("Bearer env-token",) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("row_api_base", "refused"), + [(None, False), ("https://API.mistral.ai/v1/", False), ("https://team-admin-host.example/v1", True)], + ids=["row without api_base", "row on the credential's host", "row on another host"], +) +async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials_host( + monkeypatch: pytest.MonkeyPatch, row_api_base: str | None, refused: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="mistral-sa", + credential_values={"api_key": "oidc/env/MISTRAL_SA_TOKEN", "api_base": "https://api.mistral.ai"}, + credential_info={}, + ) + ], + ) + row_params: Final = {"model": "mistral/devstral-latest", "litellm_credential_name": "mistral-sa"} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {**row_params, "api_base": row_api_base} if row_api_base else row_params, + "model_info": {"team_id": "team-1"}, + } + ], + num_retries=0, + ) + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_MISTRAL_REPLY) + if refused: + with pytest.raises(litellm.AuthenticationError, match=_CALLER_HOST_REFERENCE_REFUSAL): + await router.acompletion( + model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} + ) + else: + await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) + sent: Final = tuple( + (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls + ) + finally: + router.discard() + + assert sent == (() if refused else (("api.mistral.ai", "Bearer env-token"),)) + + +@pytest.mark.parametrize( + ("listing_params", "expected"), + [ + ( + lambda reference: LiteLLM_Params( + **litellm.Router( + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] + ).get_model_list(model_name="openai/*")[0]["litellm_params"] + ), + lambda reference: "Bearer listing-token", + ), + ( + lambda reference: LiteLLM_Params(model="openai/*", api_key=reference), + lambda reference: f"Bearer {reference}", + ), + ], + ids=["proxy-level deployment", "caller-built params"], +) +def test_provider_model_listing_sends_the_token_only_behind_a_deployment_oidc_api_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + listing_params: Callable[[str], LiteLLM_Params], + expected: Callable[[str], str], +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "listing-token") + reference: Final = f"oidc/file/{token}" + + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://api.openai.com/v1/models").respond( + 200, json={"object": "list", "data": [{"id": "gpt-listed", "object": "model"}]} + ) + listed: Final = get_valid_models( + check_provider_endpoint=True, custom_llm_provider="openai", litellm_params=listing_params(reference) + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + + assert sent == expected(reference) + assert any(model.endswith("gpt-listed") for model in listed) + + _MOCK_STREAM_ID: Final = "chatcmpl-mock-stream" _ChunkSnapshot = tuple[str, tuple[str | None, ...], Usage | None] diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 0db122b927f..e15e95f9d55 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -16,6 +16,7 @@ from litellm.types.router import ( holds_secret_pointer, reject_server_owned_wif_params, server_owned_wif_fields_named, + server_owned_wif_fields_written, server_owned_wif_fields_present, ) from litellm.types.utils import ( @@ -312,6 +313,23 @@ def test_server_owned_wif_fields_named_is_derived_from_the_shared_list(): ) +@pytest.mark.parametrize( + "api_key", ["oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", "oidc/env/SA_TOKEN", "oidc/google/aud"] +) +def test_an_oidc_reference_api_key_is_server_owned(api_key: str): + assert server_owned_wif_fields_present({"api_key": api_key}) == ("api_key",) + assert server_owned_wif_fields_written({"api_key": api_key}) == ("api_key",) + with pytest.raises(ValueError, match="api_key holds an oidc/ secret reference"): + reject_server_owned_wif_params({"api_key": api_key}) + + +@pytest.mark.parametrize("api_key", ["sk-static", "os.environ/MISTRAL_API_KEY", None]) +def test_a_plain_api_key_stays_client_settable(api_key: str | None): + assert server_owned_wif_fields_present({"api_key": api_key}) == () + assert server_owned_wif_fields_written({"api_key": api_key}) == () + reject_server_owned_wif_params({"api_key": api_key}) + + @pytest.mark.parametrize("param_name", ["anthropic_issuer_signing_key_ref", "anthropic_keycloak_client_secret_ref"]) def test_wif_ref_fields_hold_secret_pointers(param_name: str): assert holds_secret_pointer(param_name) diff --git a/tests/unit/vector_stores/test_vector_store_registry.py b/tests/unit/vector_stores/test_vector_store_registry.py index 83b59d77aaf..1a158771f29 100644 --- a/tests/unit/vector_stores/test_vector_store_registry.py +++ b/tests/unit/vector_stores/test_vector_store_registry.py @@ -20,6 +20,7 @@ from litellm.types.vector_stores import ( LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreIndex, ) +from litellm.types.utils import CredentialItem from litellm.vector_stores.main import search from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry, VectorStoreRegistry @@ -192,6 +193,125 @@ def test_search_uses_registry_credentials(): litellm.vector_store_registry = original_registry +@pytest.fixture +def registry_with_an_oidc_env_credential(monkeypatch): + monkeypatch.setenv("VS_PROJECTED_TOKEN", "projected-token-1") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="openai-sa", + credential_values={"api_key": "oidc/env/VS_PROJECTED_TOKEN", "api_base": "https://api.openai.com/v1"}, + credential_info={"custom_llm_provider": "openai"}, + ) + ], + ) + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + [ + LiteLLM_ManagedVectorStore( + vector_store_id="vs1", + custom_llm_provider="openai", + litellm_credential_name="openai-sa", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + ] + ), + ) + + +_SEARCH_PAGE: Final = { + "object": "vector_store.search_results.page", + "search_query": ["test"], + "data": [], + "has_more": False, + "next_page": None, +} +_VECTOR_STORE_FILE: Final = { + "id": "file-1", + "object": "vector_store.file", + "created_at": 0, + "vector_store_id": "vs1", + "status": "completed", + "usage_bytes": 0, + "last_error": None, +} + + +def _search_vs1(**params: str) -> None: + search(vector_store_id="vs1", query="test", custom_llm_provider="openai", **params) + + +def _create_vs1_file(**params: str) -> None: + import litellm.vector_store_files.main as vector_store_files_main + + vector_store_files_main.create(vector_store_id="vs1", file_id="file-1", **params) + + +def _sent_by(call, **params: str) -> Sequence[httpx.Request]: + with respx.mock(assert_all_called=False) as provider: + provider.post(url__regex=r"/vector_stores/vs1/search$").respond(200, json=_SEARCH_PAGE) + provider.post(url__regex=r"/vector_stores/vs1/files$").respond(200, json=_VECTOR_STORE_FILE) + call(**params) + return tuple(sent.request for sent in provider.calls) + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_a_registry_credentials_oidc_api_key_reaches_the_provider_as_the_token_it_names( + registry_with_an_oidc_env_credential, monkeypatch, call +): + (first,) = _sent_by(call) + monkeypatch.setenv("VS_PROJECTED_TOKEN", "projected-token-2") + (second,) = _sent_by(call) + + assert (first.headers["authorization"], second.headers["authorization"]) == ( + "Bearer projected-token-1", + "Bearer projected-token-2", + ) + assert litellm.credential_list[0].credential_values["api_key"] == "oidc/env/VS_PROJECTED_TOKEN" + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_a_caller_api_base_never_redirects_a_registry_credential_that_sets_its_own( + registry_with_an_oidc_env_credential, call +): + (sent,) = _sent_by(call, api_base="https://caller.example/v1") + + assert (sent.url.host, sent.headers["authorization"]) == ("api.openai.com", "Bearer projected-token-1") + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_a_registry_credential_without_an_api_base_never_sends_its_oidc_token_to_a_caller_api_base( + registry_with_an_oidc_env_credential, monkeypatch, call +): + credential: Final = litellm.credential_list[0] + monkeypatch.setattr( + litellm, + "credential_list", + [credential.model_copy(update={"credential_values": {"api_key": "oidc/env/VS_PROJECTED_TOKEN"}})], + ) + + with respx.mock(assert_all_called=False) as provider: + provider.route().respond(200, json=_SEARCH_PAGE) + with pytest.raises(litellm.AuthenticationError, match="sends it only to that credential's own api_base"): + call(api_base="https://caller.example/v1") + assert provider.calls.call_count == 0 + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_an_unresolvable_registry_credential_reference_never_reaches_the_provider( + registry_with_an_oidc_env_credential, monkeypatch, call +): + monkeypatch.delenv("VS_PROJECTED_TOKEN") + + with pytest.raises(litellm.AuthenticationError, match="oidc/env secret reference could not be resolved"): + _sent_by(call) + + def _config_registry(vector_store_id: str = "vs_from_config") -> VectorStoreRegistry: registry = VectorStoreRegistry(vector_stores=[]) registry.load_vector_stores_from_config(