diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py index ba542bebaf8..a3428261636 100644 --- a/litellm/assistants/main.py +++ b/litellm/assistants/main.py @@ -129,6 +129,7 @@ def get_assistants( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, client=client, aget_assistants=aget_assistants, ) @@ -312,6 +313,7 @@ def create_assistants( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, create_assistant_data=create_assistant_data, client=client, async_create_assistants=async_create_assistants, @@ -461,6 +463,7 @@ def delete_assistant( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, assistant_id=assistant_id, client=client, async_delete_assistants=async_delete_assistants, @@ -637,6 +640,7 @@ def create_thread( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, client=client, acreate_thread=acreate_thread, ) @@ -783,6 +787,7 @@ def get_thread( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, client=client, aget_thread=aget_thread, ) @@ -960,6 +965,7 @@ def add_message( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, client=client, a_add_message=a_add_message, ) @@ -1111,6 +1117,7 @@ def get_messages( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, client=client, aget_messages=aget_messages, ) @@ -1308,6 +1315,7 @@ def run_thread( timeout=timeout, max_retries=optional_params.max_retries, organization=organization, + litellm_params=litellm_params_dict, client=client, arun_thread=arun_thread, event_handler=event_handler, diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index e7b009eaefe..a50cdf75d33 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import ( ) from litellm.types.llms.openai import Batch from litellm.types.utils import ModelInfo, Usage -from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS +from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS from litellm.utils import token_counter @@ -569,6 +569,7 @@ def extract_file_access_credentials( # A federated deployment holds no api_key, so without these the fetch that reads a # finished batch's output has nothing to authenticate with and its cost is never billed. *sorted(ANTHROPIC_WIF_KWARGS_KEYS), + *sorted(OPENAI_WIF_KWARGS_KEYS), ) for key in credential_keys: if key in litellm_params: diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 1acd7ad67d1..9b5630d81ad 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -309,6 +309,7 @@ def create_batch( timeout=timeout, max_retries=optional_params.max_retries, _is_async=_is_async, + litellm_params=litellm_params, ) elif custom_llm_provider == "azure": api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") @@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config( organization=organization, timeout=timeout, max_retries=optional_params.max_retries, + litellm_params=litellm_params, ) elif custom_llm_provider == "azure": api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") @@ -819,6 +821,7 @@ def list_batches( organization=organization, timeout=timeout, max_retries=optional_params.max_retries, + litellm_params=litellm_params, ) elif custom_llm_provider == "azure": api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") @@ -1016,6 +1019,7 @@ def cancel_batch( organization=organization, timeout=timeout, max_retries=optional_params.max_retries, + litellm_params=litellm_params, ) elif custom_llm_provider == "azure": api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") diff --git a/litellm/containers/README.md b/litellm/containers/README.md index 571bcaf415c..40723add854 100644 --- a/litellm/containers/README.md +++ b/litellm/containers/README.md @@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig): self, headers: dict, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: """Azure uses api-key header instead of Bearer token.""" import litellm diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index ef35b61f10e..040d80e1ac0 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -458,7 +458,7 @@ def prepare_call( ) from error try: - resolved_api_base, resolved_api_key = provider_config.resolve_connection( + resolved_api_base, static_api_key = provider_config.resolve_connection( dynamic_api_base or api_base, dynamic_api_key or api_key ) except ValueError as error: @@ -469,6 +469,7 @@ def prepare_call( model=model, llm_provider=provider, ) + resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs) if resolved_api_key is None and provider_config.api_key_required: raise litellm.AuthenticationError( message=f"Missing API key for Decisions provider '{provider}'", diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 553fe4b884a..7d2ee03da0e 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,5 +1,7 @@ +import asyncio import warnings from collections.abc import Mapping +from functools import partial from typing import Final import httpx @@ -104,19 +106,22 @@ async def adecisions( videos: DecisionsVideoClips | None = None, **kwargs: object, ) -> OpenAIDecisionResponse | DecisionsResponse: - call: Final = _prepare_decisions_call( - model=model, - decision_input=input, - questions=questions, - safety_identifier=safety_identifier, - audio=audio, - videos=videos, - api_key=api_key, - api_base=api_base, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - kwargs=kwargs, + call: Final = await asyncio.to_thread( + partial( + _prepare_decisions_call, + model=model, + decision_input=input, + questions=questions, + safety_identifier=safety_identifier, + audio=audio, + videos=videos, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) ) return await asend(call) diff --git a/litellm/files/main.py b/litellm/files/main.py index e259d4230de..16ff5b02f9c 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -259,6 +259,7 @@ def create_file( max_retries=optional_params.max_retries, organization=openai_creds.organization, create_file_data=_create_file_request, + litellm_params=litellm_params_dict, ) elif custom_llm_provider == "azure": azure_creds: Final = get_azure_credentials( @@ -388,6 +389,7 @@ def file_retrieve( timeout=timeout, max_retries=optional_params.max_retries, organization=openai_creds.organization, + litellm_params=optional_params.model_dump(exclude_none=True), ) elif custom_llm_provider == "azure": azure_creds: Final = get_azure_credentials( @@ -569,6 +571,7 @@ def file_delete( timeout=timeout, max_retries=optional_params.max_retries, organization=openai_creds.organization, + litellm_params=litellm_params_dict, ) elif custom_llm_provider == "azure": azure_creds: Final = get_azure_credentials( @@ -775,6 +778,7 @@ def file_list( timeout=timeout, max_retries=optional_params.max_retries, organization=openai_creds.organization, + litellm_params=optional_params.model_dump(exclude_none=True), ) elif custom_llm_provider == "azure": azure_creds: Final = get_azure_credentials( @@ -972,6 +976,7 @@ def file_content( timeout=timeout, max_retries=optional_params.max_retries, organization=openai_creds.organization, + litellm_params=litellm_params_dict, ) elif custom_llm_provider == "azure": azure_creds: Final = get_azure_credentials( @@ -1093,6 +1098,7 @@ def file_content_streaming( organization=openai_creds.organization, chunk_size=chunk_size, client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None, + litellm_params=optional_params.model_dump(exclude_none=True), ) elif custom_llm_provider == LlmProviders.VERTEX_AI.value: if not _is_async: diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 38be0666008..5297a8215be 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -239,6 +239,7 @@ def create_fine_tuning_job( client=kwargs.get( "client", None ), # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting + litellm_params=optional_params.model_dump(exclude_none=True), ) # Azure OpenAI elif custom_llm_provider == "azure": @@ -436,6 +437,7 @@ def cancel_fine_tuning_job( max_retries=optional_params.max_retries, _is_async=_is_async, client=kwargs.get("client", None), + litellm_params=optional_params.model_dump(exclude_none=True), ) # Azure OpenAI elif custom_llm_provider == "azure": @@ -591,6 +593,7 @@ def list_fine_tuning_jobs( max_retries=optional_params.max_retries, _is_async=_is_async, client=kwargs.get("client", None), + litellm_params=optional_params.model_dump(exclude_none=True), ) # Azure OpenAI elif custom_llm_provider == "azure": @@ -730,6 +733,7 @@ def retrieve_fine_tuning_job( max_retries=optional_params.max_retries, _is_async=_is_async, client=kwargs.get("client", None), + litellm_params=optional_params.model_dump(exclude_none=True), ) # Azure OpenAI elif custom_llm_provider == "azure": diff --git a/litellm/images/main.py b/litellm/images/main.py index be32ea4daed..7f94835d59b 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes # BFL handlers from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation +from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config from litellm.main import ( azure_chat_completions, base_llm_aiohttp_handler, @@ -505,6 +506,7 @@ def image_generation( aimg_generation=aimg_generation, client=client, headers=headers, + litellm_params=litellm_params_dict, ) elif custom_llm_provider == "bedrock": if model is None: @@ -681,10 +683,13 @@ def image_variation( api_base = provider_config.get_api_base(litellm_params.get("api_base", None)) if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI: - if api_key is None: - raise ValueError("API key is required for OpenAI image variations") if api_base is None: raise ValueError("API base is required for OpenAI image variations") + if api_key is None and ( + resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params) + is None + ): + raise ValueError("API key is required for OpenAI image variations") response = openai_image_variations.image_variations( model_response=model_response, diff --git a/litellm/llms/azure/containers/transformation.py b/litellm/llms/azure/containers/transformation.py index 5031f38bf8f..d69c39d8873 100644 --- a/litellm/llms/azure/containers/transformation.py +++ b/litellm/llms/azure/containers/transformation.py @@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig): self, headers: dict, api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, ) -> dict: return BaseAzureLLM.base_validate_azure_environment( headers=headers, diff --git a/litellm/llms/azure/fine_tuning/handler.py b/litellm/llms/azure/fine_tuning/handler.py index 0baf3ccc598..aa410d8232b 100644 --- a/litellm/llms/azure/fine_tuning/handler.py +++ b/litellm/llms/azure/fine_tuning/handler.py @@ -1,4 +1,4 @@ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import Final, cast import httpx @@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: self._ensure_training_type(create_fine_tuning_job_data) @@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM): client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, _is_async: bool = False, api_version: str | None = None, - litellm_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None: # Override to use Azure-specific client initialization if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI): client = None return self.get_azure_openai_client( - litellm_params=litellm_params or {}, + litellm_params=dict(litellm_params) if litellm_params is not None else None, api_key=api_key, api_base=api_base, api_version=api_version, diff --git a/litellm/llms/base_llm/containers/transformation.py b/litellm/llms/base_llm/containers/transformation.py index 49487cddb6f..7a04c4d9488 100644 --- a/litellm/llms/base_llm/containers/transformation.py +++ b/litellm/llms/base_llm/containers/transformation.py @@ -81,6 +81,7 @@ class BaseContainerConfig(ABC): self, headers: dict, api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, ) -> dict: return {} diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 041473cb634..df7b7645840 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -495,6 +495,11 @@ class BaseDecisionsConfig(ABC): def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]: return self.resolve_api_base(api_base), self.resolve_api_key(api_key) + def resolve_credential( + self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object] + ) -> str | None: + return api_key + @staticmethod def _first_secret(names: tuple[str, ...]) -> str | None: return next((value for value in (get_secret_str(name) for name in names) if value), None) diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index bfa163c7546..a9b96217379 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -5,6 +5,7 @@ This module provides a single generic handler that can process any container fil endpoint defined in endpoints.json, eliminating the need for individual handler methods. """ +import asyncio import json from collections.abc import Coroutine, Mapping, Sequence from pathlib import Path @@ -236,6 +237,7 @@ def _request_headers( return container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) @@ -435,7 +437,7 @@ class GenericContainerHandler: http_client: Final = _async_http_client(client, litellm_params) # Build request - headers = _request_headers(container_provider_config, extra_headers, litellm_params) + headers = await asyncio.to_thread(_request_headers, container_provider_config, extra_headers, litellm_params) if extra_headers: headers.update(extra_headers) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 610321fe1f6..38e076d650a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -7127,7 +7127,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = image_edit_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + image_edit_provider_config.validate_environment, api_key=litellm_params.api_key, headers=image_edit_optional_request_params.get("extra_headers", {}) or {}, model=model, @@ -7586,7 +7587,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_generation_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_generation_provider_config.validate_environment, api_key=api_key or litellm_params.get("api_key", None), headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, @@ -7781,7 +7783,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_content_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_content_provider_config.validate_environment, headers=extra_headers or {}, model="", api_key=api_key, @@ -7961,7 +7964,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_remix_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_remix_provider_config.validate_environment, api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", @@ -8124,7 +8128,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_provider_config.validate_environment, api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", @@ -8264,7 +8269,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_provider_config.validate_environment, api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", @@ -8436,7 +8442,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_provider_config.validate_environment, api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", @@ -8622,7 +8629,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_provider_config.validate_environment, api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", @@ -8754,7 +8762,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_list_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_list_provider_config.validate_environment, api_key=api_key, headers=extra_headers or {}, model="", @@ -8835,7 +8844,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_delete_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_delete_provider_config.validate_environment, api_key=api_key, headers=extra_headers or {}, model="", @@ -9019,7 +9029,8 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = video_status_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + video_status_provider_config.validate_environment, api_key=api_key, headers=extra_headers or {}, model="", @@ -9122,6 +9133,7 @@ class BaseLLMHTTPHandler: headers: Final = container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) # Add Content-Type header for JSON requests @@ -9197,9 +9209,11 @@ class BaseLLMHTTPHandler: async_httpx_client = client # Validate environment and get headers - headers: Final = container_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + container_provider_config.validate_environment, headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) # Add Content-Type header for JSON requests @@ -9293,6 +9307,7 @@ class BaseLLMHTTPHandler: headers: Final = container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9369,9 +9384,11 @@ class BaseLLMHTTPHandler: async_httpx_client = client # Validate environment and get headers - headers: Final = container_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + container_provider_config.validate_environment, headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9460,6 +9477,7 @@ class BaseLLMHTTPHandler: headers: Final = container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9536,9 +9554,11 @@ class BaseLLMHTTPHandler: async_httpx_client = client # Validate environment and get headers - headers: Final = container_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + container_provider_config.validate_environment, headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9629,6 +9649,7 @@ class BaseLLMHTTPHandler: headers: Final = container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9705,9 +9726,11 @@ class BaseLLMHTTPHandler: async_httpx_client = client # Validate environment and get headers - headers: Final = container_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + container_provider_config.validate_environment, headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9803,6 +9826,7 @@ class BaseLLMHTTPHandler: headers: Final = container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9881,9 +9905,11 @@ class BaseLLMHTTPHandler: async_httpx_client = client # Validate environment and get headers - headers: Final = container_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + container_provider_config.validate_environment, headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -9972,6 +9998,7 @@ class BaseLLMHTTPHandler: headers: Final = container_provider_config.validate_environment( headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -10047,9 +10074,11 @@ class BaseLLMHTTPHandler: async_httpx_client = client # Validate environment and get headers - headers: Final = container_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + container_provider_config.validate_environment, headers=extra_headers or {}, api_key=litellm_params.get("api_key", None), + litellm_params=litellm_params, ) if extra_headers: @@ -10167,8 +10196,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = vector_store_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers = await asyncio.to_thread( + vector_store_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -10410,8 +10441,10 @@ class BaseLLMHTTPHandler: vector_store_provider_config.validate_create_vector_store() - headers: Final = vector_store_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -10541,8 +10574,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -10663,8 +10698,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -10805,8 +10842,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -10950,8 +10989,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -11068,8 +11109,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_files_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_files_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: @@ -11210,8 +11253,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_files_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_files_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -11351,8 +11396,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_files_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_files_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -11482,8 +11529,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_files_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_files_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -11619,8 +11668,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_files_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_files_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -11762,8 +11813,10 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = vector_store_files_provider_config.validate_environment( - headers=extra_headers or {}, litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + vector_store_files_provider_config.validate_environment, + headers=extra_headers or {}, + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 4ab91392ad4..e472661efb9 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -407,6 +407,10 @@ class BaseOpenAILLM: http2=http2_enabled(), ) + @classmethod + def get_sync_http_client(cls) -> httpx.Client | None: + return cls._get_sync_http_client() + class OpenAICredentials(NamedTuple): api_base: str diff --git a/litellm/llms/openai/completion/handler.py b/litellm/llms/openai/completion/handler.py index 174aefd29e5..278427e8d77 100644 --- a/litellm/llms/openai/completion/handler.py +++ b/litellm/llms/openai/completion/handler.py @@ -1,9 +1,7 @@ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final -from openai import AsyncOpenAI, OpenAI - import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper @@ -12,7 +10,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse from litellm.utils import ProviderConfigManager -from ..common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient +from ..common_utils import BaseOpenAILLM, OpenAIError +from ..workload_identity import build_async_openai_client, build_openai_client from .transformation import OpenAITextCompletionConfig @@ -43,7 +42,7 @@ class OpenAITextCompletion(BaseLLM): print_verbose: Callable | None = None, api_base: str | None = None, acompletion: bool = False, - litellm_params=None, + litellm_params: Mapping[str, object] | None = None, logger_fn=None, client=None, organization: str | None = None, @@ -95,6 +94,7 @@ class OpenAITextCompletion(BaseLLM): max_retries=max_retries, client=client, organization=organization, + litellm_params=litellm_params, ) else: return self.acompletion( @@ -109,6 +109,7 @@ class OpenAITextCompletion(BaseLLM): max_retries=max_retries, organization=organization, client=client, + litellm_params=litellm_params, ) elif optional_params.get("stream", False): return self.streaming( @@ -123,19 +124,22 @@ class OpenAITextCompletion(BaseLLM): max_retries=max_retries, client=client, organization=organization, + litellm_params=litellm_params, ) else: - if client is None: - openai_client = OpenAI( + openai_client: Final = ( + client + if client is not None + else build_openai_client( api_key=api_key, - base_url=api_base, - http_client=litellm.client_session or OpenAIHTTPClient(), + api_base=api_base, timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, + static_key_http_client_factory=lambda: litellm.client_session, ) - else: - openai_client = client + ) raw_response: Final = openai_client.completions.with_raw_response.create(**data) response: Final = raw_response.parse() @@ -175,19 +179,22 @@ class OpenAITextCompletion(BaseLLM): max_retries: int, organization: str | None = None, client=None, + litellm_params: Mapping[str, object] | None = None, ): try: - if client is None: - openai_aclient = AsyncOpenAI( + openai_aclient: Final = ( + client + if client is not None + else build_async_openai_client( api_key=api_key, - base_url=api_base, - http_client=BaseOpenAILLM.get_async_http_client(), + api_base=api_base, timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, + static_key_http_client_factory=BaseOpenAILLM.get_async_http_client, ) - else: - openai_aclient = client + ) raw_response: Final = await openai_aclient.completions.with_raw_response.create(**data) response: Final = raw_response.parse() @@ -228,18 +235,21 @@ class OpenAITextCompletion(BaseLLM): max_retries=None, client=None, organization=None, + litellm_params: Mapping[str, object] | None = None, ): - if client is None: - openai_client = OpenAI( + openai_client: Final = ( + client + if client is not None + else build_openai_client( api_key=api_key, - base_url=api_base, - http_client=litellm.client_session or OpenAIHTTPClient(), + api_base=api_base, timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, + static_key_http_client_factory=lambda: litellm.client_session, ) - else: - openai_client = client + ) try: raw_response: Final = openai_client.completions.with_raw_response.create(**data) @@ -285,18 +295,21 @@ class OpenAITextCompletion(BaseLLM): api_base: str | None = None, client=None, organization=None, + litellm_params: Mapping[str, object] | None = None, ): - if client is None: - openai_client = AsyncOpenAI( + openai_client: Final = ( + client + if client is not None + else build_async_openai_client( api_key=api_key, - base_url=api_base, - http_client=litellm.aclient_session or OpenAIAsyncHTTPClient(), + api_base=api_base, timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, + static_key_http_client_factory=lambda: litellm.aclient_session, ) - else: - openai_client = client + ) raw_response: Final = await openai_client.completions.with_raw_response.create(**data) response: Final = raw_response.parse() diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py index bff5e0cea10..6cb2396b85d 100644 --- a/litellm/llms/openai/containers/transformation.py +++ b/litellm/llms/openai/containers/transformation.py @@ -23,6 +23,7 @@ from litellm.types.containers.main import ( from litellm.types.router import GenericLiteLLMParams from ...base_llm.containers.transformation import BaseContainerConfig +from ..workload_identity import resolve_openai_bearer_token from .utils import join_container_api_base_path if TYPE_CHECKING: @@ -104,13 +105,15 @@ class OpenAIContainerConfig(BaseContainerConfig): self, headers: dict, api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, ) -> dict: - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - headers.update( - { - "Authorization": f"Bearer {api_key}", - }, + params: Final = litellm_params or GenericLiteLLMParams() + bearer_token: Final = resolve_openai_bearer_token( + api_key=api_key or params.api_key, + api_base=params.api_base, + litellm_params=params.model_dump(exclude_none=True), ) + headers.update({"Authorization": f"Bearer {bearer_token}"}) return headers def get_complete_url( diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 7ce5b8676ee..56f348881de 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -6,6 +6,7 @@ from typing_extensions import assert_never import litellm from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text +from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( DecisionsImage, @@ -349,6 +350,11 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig): def resolve_api_key(self, api_key: str | None) -> str | None: return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + def resolve_credential( + self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object] + ) -> str | None: + return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params) + def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: return False diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 7980c5529cb..24de7f6d479 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -13,6 +13,7 @@ from litellm.llms.base_llm.evals.transformation import ( BaseEvalsAPIConfig, LiteLLMLoggingObj, ) +from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -44,24 +45,14 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): return LlmProviders.OPENAI def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict: - """Add OpenAI-specific headers""" - import litellm - from litellm.secret_managers.main import get_secret_str - - # Get API key following OpenAI pattern - api_key = None - if litellm_params: - api_key = litellm_params.api_key - - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - - if not api_key: + params: Final = litellm_params or GenericLiteLLMParams() + bearer_token: Final = resolve_openai_bearer_token( + api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True) + ) + if not bearer_token: raise ValueError("OPENAI_API_KEY is required for Evals API") - - # Add required headers - headers["Authorization"] = f"Bearer {api_key}" + headers["Authorization"] = f"Bearer {bearer_token}" headers["Content-Type"] = "application/json" - return headers def get_complete_url( diff --git a/litellm/llms/openai/fine_tuning/handler.py b/litellm/llms/openai/fine_tuning/handler.py index 851113bbd37..9ba28803f93 100644 --- a/litellm/llms/openai/fine_tuning/handler.py +++ b/litellm/llms/openai/fine_tuning/handler.py @@ -6,7 +6,7 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from openai.types.fine_tuning import FineTuningJob from litellm._logging import verbose_logger -from litellm.llms.openai.common_utils import OpenAIAsyncHTTPClient, OpenAIHTTPClient +from litellm.llms.openai.workload_identity import build_async_openai_client, build_openai_client from litellm.types.utils import LiteLLMFineTuningJob _AZURE_STATUS_MAP: Final[Mapping[object, str]] = { @@ -74,27 +74,27 @@ class OpenAIFineTuningAPI: client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, _is_async: bool = False, api_version: str | None = None, - litellm_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None: - received_args: Final = locals() - openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None - if client is None: - data: Final = {} - for k, v in received_args.items(): - if k == "self" or k == "client" or k == "_is_async": - pass - elif k == "api_base" and v is not None: - data["base_url"] = v - elif v is not None: - data[k] = v - if _is_async is True: - openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient()) - else: - openai_client = OpenAI(**data, http_client=OpenAIHTTPClient()) - else: - openai_client = client - - return openai_client + if client is not None: + return client + if _is_async: + return build_async_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) + return build_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) async def acreate_fine_tuning_job( self, @@ -116,6 +116,7 @@ class OpenAIFineTuningAPI: max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -126,6 +127,7 @@ class OpenAIFineTuningAPI: client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -164,6 +166,7 @@ class OpenAIFineTuningAPI: max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -174,6 +177,7 @@ class OpenAIFineTuningAPI: client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -212,6 +216,7 @@ class OpenAIFineTuningAPI: max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, after: str | None = None, limit: int | None = None, ): @@ -224,6 +229,7 @@ class OpenAIFineTuningAPI: client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -263,6 +269,7 @@ class OpenAIFineTuningAPI: max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]: openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -273,6 +280,7 @@ class OpenAIFineTuningAPI: client=client, _is_async=_is_async, api_version=api_version, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( diff --git a/litellm/llms/openai/image_edit/transformation.py b/litellm/llms/openai/image_edit/transformation.py index d54522597a0..a9e5b1b9f34 100644 --- a/litellm/llms/openai/image_edit/transformation.py +++ b/litellm/llms/openai/image_edit/transformation.py @@ -17,6 +17,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import ImageResponse from ..common_utils import OpenAIError +from ..workload_identity import resolve_openai_bearer_token if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -160,12 +161,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig): litellm_params: dict | None = None, api_base: str | None = None, ) -> dict: - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - headers.update( - { - "Authorization": f"Bearer {api_key}", - } + bearer_token: Final = resolve_openai_bearer_token( + api_key=api_key, api_base=api_base, litellm_params=litellm_params ) + headers.update({"Authorization": f"Bearer {bearer_token}"}) return headers def get_complete_url( diff --git a/litellm/llms/openai/image_variations/handler.py b/litellm/llms/openai/image_variations/handler.py index 664d16c48bf..89dcdc40e02 100644 --- a/litellm/llms/openai/image_variations/handler.py +++ b/litellm/llms/openai/image_variations/handler.py @@ -2,7 +2,7 @@ OpenAI Image Variations Handler """ -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import httpx @@ -14,27 +14,58 @@ from litellm.utils import ProviderConfigManager from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj -from ..common_utils import OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient +from ..common_utils import OpenAIError +from ..workload_identity import build_async_openai_client, build_openai_client class OpenAIImageVariationsHandler: def get_sync_client( self, client: OpenAI | None, - init_client_params: dict, - ): + api_key: str | None, + api_base: str, + timeout: float | httpx.Timeout | None, + max_retries: int, + organization: str | None, + litellm_params: Mapping[str, object], + ) -> OpenAI: if client is not None: return client - return OpenAI(**init_client_params, http_client=litellm.client_session or OpenAIHTTPClient()) + return build_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + static_key_http_client_factory=lambda: litellm.client_session, + ) - def get_async_client(self, client: AsyncOpenAI | None, init_client_params: dict) -> AsyncOpenAI: + def get_async_client( + self, + client: AsyncOpenAI | None, + api_key: str | None, + api_base: str, + timeout: float | httpx.Timeout | None, + max_retries: int, + organization: str | None, + litellm_params: Mapping[str, object], + ) -> AsyncOpenAI: if client is not None: return client - return AsyncOpenAI(**init_client_params, http_client=litellm.aclient_session or OpenAIAsyncHTTPClient()) + return build_async_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + static_key_http_client_factory=lambda: litellm.aclient_session, + ) async def async_image_variations( self, - api_key: str, + api_key: str | None, api_base: str, organization: str | None, client: AsyncOpenAI | None, @@ -51,15 +82,15 @@ class OpenAIImageVariationsHandler: provider_config: BaseImageVariationConfig, ) -> ImageResponse: try: - init_client_params: Final = { - "api_key": api_key, - "base_url": api_base, - "timeout": timeout, - "max_retries": max_retries, - "organization": organization, - } - - client = self.get_async_client(client=client, init_client_params=init_client_params) + client = self.get_async_client( + client=client, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) raw_response: Final = await client.images.with_raw_response.create_variation(**data) response: Final = raw_response.parse() @@ -103,7 +134,7 @@ class OpenAIImageVariationsHandler: def image_variations( self, model_response: ImageResponse, - api_key: str, + api_key: str | None, api_base: str, model: str | None, image: FileTypes, @@ -167,15 +198,15 @@ class OpenAIImageVariationsHandler: litellm_params=litellm_params, ) - init_client_params: Final = { - "api_key": api_key, - "base_url": api_base, - "timeout": timeout, - "max_retries": max_retries, - "organization": organization, - } - - client = self.get_sync_client(client=client, init_client_params=init_client_params) + client = self.get_sync_client( + client=client, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) raw_response: Final = client.images.with_raw_response.create_variation(**json_data) response: Final = raw_response.parse() diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 79c69b5ceb4..0c1eaaaae26 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -52,16 +52,19 @@ from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, - OpenAIAsyncHTTPClient, OpenAIError, - OpenAIHTTPClient, build_output_token_limit_response, drop_params_from_unprocessable_entity_error, is_openai_backed_api_base, is_output_token_limit_error, model_dump_provider_response, ) -from .workload_identity import resolve_openai_workload_identity_config +from .workload_identity import ( + build_async_openai_client, + build_openai_client, + openai_static_api_key, + resolve_openai_workload_identity_config, +) openaiOSeriesConfig: Final = OpenAIOSeriesConfig() openAIGPT5Config: Final = OpenAIGPT5Config() @@ -425,7 +428,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) if workload_identity_config is not None else AsyncOpenAI( - api_key=api_key, + api_key=openai_static_api_key(api_key), base_url=api_base, http_client=async_http_client, timeout=timeout, @@ -447,7 +450,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) if workload_identity_config is not None else OpenAI( - api_key=api_key, + api_key=openai_static_api_key(api_key), base_url=api_base, http_client=sync_http_client, timeout=timeout, @@ -1455,10 +1458,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): max_retries=None, organization: str | None = None, headers: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ): try: openai_aclient: Final = self._get_openai_client( is_async=True, + litellm_params=litellm_params, api_key=api_key, api_base=api_base, timeout=timeout, @@ -1520,6 +1525,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): aimg_generation=None, organization: str | None = None, headers: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> ImageResponse: data = {} try: @@ -1541,10 +1547,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): max_retries=max_retries, organization=organization, headers=headers, + litellm_params=litellm_params, ) openai_client: Final[OpenAI] = self._get_openai_client( is_async=False, + litellm_params=litellm_params, api_key=api_key, api_base=api_base, timeout=timeout, @@ -1622,6 +1630,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): aspeech: bool | None = None, client=None, shared_session: Optional["ClientSession"] = None, + litellm_params: Mapping[str, object] | None = None, ) -> HttpxBinaryResponseContent: if aspeech is not None and aspeech is True: return self.async_audio_speech( @@ -1638,6 +1647,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj=logging_obj, client=client, shared_session=shared_session, + litellm_params=litellm_params, ) openai_client: Final = self._get_openai_client( @@ -1648,6 +1658,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): max_retries=max_retries, client=client, shared_session=shared_session, + litellm_params=litellm_params, ) sync_client: Final = cast(OpenAI, openai_client) @@ -1685,6 +1696,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj: LiteLLMLoggingObj, client=None, shared_session: Optional["ClientSession"] = None, + litellm_params: Mapping[str, object] | None = None, ) -> HttpxBinaryResponseContent: openai_client: Final = cast( AsyncOpenAI, @@ -1696,6 +1708,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): max_retries=max_retries, client=client, shared_session=shared_session, + litellm_params=litellm_params, ), ) @@ -1742,26 +1755,27 @@ class OpenAIFilesAPI(BaseLLM): organization: str | None, client: OpenAI | AsyncOpenAI | None = None, _is_async: bool = False, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAI | AsyncOpenAI | None: - received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) - openai_client: OpenAI | AsyncOpenAI | None = None - if client is None: - data: Final = {} - for k, v in received_args.items(): - if k == "self" or k == "client" or k == "_is_async": - pass - elif k == "api_base" and v is not None: - data["base_url"] = v - elif v is not None: - data[k] = v - if _is_async is True: - openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient()) - else: - openai_client = OpenAI(**data, http_client=OpenAIHTTPClient()) - else: - openai_client = client - - return openai_client + if client is not None: + return client + if _is_async: + return build_async_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) + return build_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) async def acreate_file( self, @@ -1781,6 +1795,7 @@ class OpenAIFilesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAIFileObject | Coroutine[None, None, OpenAIFileObject]: openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -1790,6 +1805,7 @@ class OpenAIFilesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -1823,6 +1839,7 @@ class OpenAIFilesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]: openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -1832,6 +1849,7 @@ class OpenAIFilesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -1888,6 +1906,7 @@ class OpenAIFilesAPI(BaseLLM): organization: str | None, chunk_size: int = 1024 * 1024, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> FileContentStreamingResult: openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -1897,6 +1916,7 @@ class OpenAIFilesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -1951,6 +1971,7 @@ class OpenAIFilesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -1960,6 +1981,7 @@ class OpenAIFilesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -1997,6 +2019,7 @@ class OpenAIFilesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -2006,6 +2029,7 @@ class OpenAIFilesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -2046,6 +2070,7 @@ class OpenAIFilesAPI(BaseLLM): organization: str | None, purpose: str | None = None, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -2055,6 +2080,7 @@ class OpenAIFilesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -2100,26 +2126,27 @@ class OpenAIBatchesAPI(BaseLLM): organization: str | None, client: OpenAI | AsyncOpenAI | None = None, _is_async: bool = False, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAI | AsyncOpenAI | None: - received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) - openai_client: OpenAI | AsyncOpenAI | None = None - if client is None: - data: Final = {} - for k, v in received_args.items(): - if k == "self" or k == "client" or k == "_is_async": - pass - elif k == "api_base" and v is not None: - data["base_url"] = v - elif v is not None: - data[k] = v - if _is_async is True: - openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient()) - else: - openai_client = OpenAI(**data, http_client=OpenAIHTTPClient()) - else: - openai_client = client - - return openai_client + if client is not None: + return client + if _is_async: + return build_async_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) + return build_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) async def acreate_batch( self, @@ -2139,6 +2166,7 @@ class OpenAIBatchesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | AsyncOpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]: openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -2148,6 +2176,7 @@ class OpenAIBatchesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -2183,6 +2212,7 @@ class OpenAIBatchesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -2192,6 +2222,7 @@ class OpenAIBatchesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -2226,6 +2257,7 @@ class OpenAIBatchesAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -2235,6 +2267,7 @@ class OpenAIBatchesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -2275,6 +2308,7 @@ class OpenAIBatchesAPI(BaseLLM): after: str | None = None, limit: int | None = None, client: OpenAI | None = None, + litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( api_key=api_key, @@ -2284,6 +2318,7 @@ class OpenAIBatchesAPI(BaseLLM): organization=organization, client=client, _is_async=_is_async, + litellm_params=litellm_params, ) if openai_client is None: raise ValueError( @@ -2312,22 +2347,19 @@ class OpenAIAssistantsAPI(BaseLLM): max_retries: int | None, organization: str | None, client: OpenAI | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAI: - received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) - if client is None: - data: Final = {} - for k, v in received_args.items(): - if k == "self" or k == "client": - pass - elif k == "api_base" and v is not None: - data["base_url"] = v - elif v is not None: - data[k] = v - openai_client = OpenAI(**data, http_client=OpenAIHTTPClient()) - else: - openai_client = client - - return openai_client + if client is not None: + return client + return build_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) def async_get_openai_client( self, @@ -2337,22 +2369,19 @@ class OpenAIAssistantsAPI(BaseLLM): max_retries: int | None, organization: str | None, client: AsyncOpenAI | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> AsyncOpenAI: - received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) - if client is None: - data: Final = {} - for k, v in received_args.items(): - if k == "self" or k == "client": - pass - elif k == "api_base" and v is not None: - data["base_url"] = v - elif v is not None: - data[k] = v - openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient()) - else: - openai_client = client - - return openai_client + if client is not None: + return client + return build_async_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + litellm_params=litellm_params, + ) ### ASSISTANTS ### @@ -2368,6 +2397,8 @@ class OpenAIAssistantsAPI(BaseLLM): limit: int | None = 20, before: str | None = None, after: str | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> AsyncCursorPage[Assistant]: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2375,6 +2406,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) request_params: Final = { @@ -2402,6 +2434,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, aget_assistants: Literal[True], + *, + litellm_params: Mapping[str, object] | None = None, ) -> Coroutine[None, None, AsyncCursorPage[Assistant]]: ... @@ -2415,6 +2449,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: OpenAI | None, aget_assistants: Literal[False] | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> SyncCursorPage[Assistant]: ... @@ -2433,7 +2469,9 @@ class OpenAIAssistantsAPI(BaseLLM): limit: int | None = 20, before: str | None = None, after: str | None = None, - ): + *, + litellm_params: Mapping[str, object] | None = None, + ) -> Coroutine[None, None, AsyncCursorPage[Assistant]] | SyncCursorPage[Assistant]: if aget_assistants is not None and aget_assistants is True: return self.async_get_assistants( api_key=api_key, @@ -2441,6 +2479,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) openai_client: Final = self.get_openai_client( @@ -2449,6 +2488,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2476,6 +2516,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, create_assistant_data: dict, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Assistant: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2483,6 +2525,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2500,6 +2543,8 @@ class OpenAIAssistantsAPI(BaseLLM): create_assistant_data: dict, client=None, async_create_assistants=None, + *, + litellm_params: Mapping[str, object] | None = None, ): if async_create_assistants is not None and async_create_assistants is True: return self.async_create_assistants( @@ -2508,6 +2553,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, create_assistant_data=create_assistant_data, ) @@ -2517,6 +2563,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2533,6 +2580,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, assistant_id: str, + *, + litellm_params: Mapping[str, object] | None = None, ) -> AssistantDeleted: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2540,6 +2589,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2557,6 +2607,8 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, client=None, async_delete_assistants=None, + *, + litellm_params: Mapping[str, object] | None = None, ): if async_delete_assistants is not None and async_delete_assistants is True: return self.async_delete_assistant( @@ -2565,6 +2617,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, assistant_id=assistant_id, ) @@ -2574,6 +2627,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2592,6 +2646,8 @@ class OpenAIAssistantsAPI(BaseLLM): max_retries: int | None, organization: str | None, client: AsyncOpenAI | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAIMessage: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2599,6 +2655,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2629,6 +2686,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, a_add_message: Literal[True], + *, + litellm_params: Mapping[str, object] | None = None, ) -> Coroutine[None, None, OpenAIMessage]: ... @@ -2644,6 +2703,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: OpenAI | None, a_add_message: Literal[False] | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> OpenAIMessage: ... @@ -2660,6 +2721,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client=None, a_add_message: bool | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ): if a_add_message is not None and a_add_message is True: return self.a_add_message( @@ -2670,6 +2733,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) openai_client: Final = self.get_openai_client( @@ -2678,6 +2742,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2703,6 +2768,8 @@ class OpenAIAssistantsAPI(BaseLLM): max_retries: int | None, organization: str | None, client: AsyncOpenAI | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> AsyncCursorPage[OpenAIMessage]: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2710,6 +2777,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2730,6 +2798,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, aget_messages: Literal[True], + *, + litellm_params: Mapping[str, object] | None = None, ) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]: ... @@ -2744,6 +2814,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: OpenAI | None, aget_messages: Literal[False] | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> SyncCursorPage[OpenAIMessage]: ... @@ -2759,6 +2831,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client=None, aget_messages=None, + *, + litellm_params: Mapping[str, object] | None = None, ): if aget_messages is not None and aget_messages is True: return self.async_get_messages( @@ -2768,6 +2842,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) openai_client: Final = self.get_openai_client( @@ -2776,6 +2851,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2795,6 +2871,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, messages: Iterable[OpenAICreateThreadParamsMessage] | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Thread: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2802,6 +2880,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2834,6 +2913,8 @@ class OpenAIAssistantsAPI(BaseLLM): messages: Iterable[OpenAICreateThreadParamsMessage] | None, client: AsyncOpenAI | None, acreate_thread: Literal[True], + *, + litellm_params: Mapping[str, object] | None = None, ) -> Coroutine[None, None, Thread]: ... @@ -2849,6 +2930,8 @@ class OpenAIAssistantsAPI(BaseLLM): messages: Iterable[OpenAICreateThreadParamsMessage] | None, client: OpenAI | None, acreate_thread: Literal[False] | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Thread: ... @@ -2865,6 +2948,8 @@ class OpenAIAssistantsAPI(BaseLLM): messages: Iterable[OpenAICreateThreadParamsMessage] | None, client=None, acreate_thread=None, + *, + litellm_params: Mapping[str, object] | None = None, ): """ Here's an example: @@ -2884,6 +2969,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, messages=messages, ) @@ -2893,6 +2979,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2920,6 +3007,8 @@ class OpenAIAssistantsAPI(BaseLLM): max_retries: int | None, organization: str | None, client: AsyncOpenAI | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Thread: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -2927,6 +3016,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -2952,6 +3042,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None, aget_thread: Literal[True], + *, + litellm_params: Mapping[str, object] | None = None, ) -> Coroutine[None, None, Thread]: ... @@ -2966,6 +3058,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: OpenAI | None, aget_thread: Literal[False] | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Thread: ... @@ -2981,6 +3075,8 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client=None, aget_thread=None, + *, + litellm_params: Mapping[str, object] | None = None, ): if aget_thread is not None and aget_thread is True: return self.async_get_thread( @@ -2990,6 +3086,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) openai_client: Final = self.get_openai_client( @@ -2998,6 +3095,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -3031,6 +3129,8 @@ class OpenAIAssistantsAPI(BaseLLM): max_retries: int | None, organization: str | None, client: AsyncOpenAI | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Run: openai_client: Final = self.async_get_openai_client( api_key=api_key, @@ -3038,6 +3138,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) @@ -3142,6 +3243,8 @@ class OpenAIAssistantsAPI(BaseLLM): client, arun_thread: Literal[True], event_handler: AssistantEventHandler | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Coroutine[None, None, Run]: ... @@ -3164,6 +3267,8 @@ class OpenAIAssistantsAPI(BaseLLM): client, arun_thread: Literal[False] | None, event_handler: AssistantEventHandler | None, + *, + litellm_params: Mapping[str, object] | None = None, ) -> Run: ... @@ -3187,6 +3292,8 @@ class OpenAIAssistantsAPI(BaseLLM): client=None, arun_thread=None, event_handler: AssistantEventHandler | None = None, + *, + litellm_params: Mapping[str, object] | None = None, ): if arun_thread is not None and arun_thread is True: if stream is not None and stream is True: @@ -3196,6 +3303,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) return self.async_run_thread_stream( @@ -3223,6 +3331,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) openai_client: Final = self.get_openai_client( @@ -3231,6 +3340,7 @@ class OpenAIAssistantsAPI(BaseLLM): timeout=timeout, max_retries=max_retries, organization=organization, + litellm_params=litellm_params, client=client, ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 274d693baa7..c7c997bc961 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -27,6 +27,7 @@ from litellm.llms.openai.responses.count_tokens.transformation import ( from litellm.llms.openai.workload_identity import ( OpenAIWorkloadIdentityConfig, get_workload_identity_bearer_token, + openai_static_api_key, resolve_openai_workload_identity_config, ) from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse @@ -35,13 +36,18 @@ from litellm.types.utils import CountContextManagement, LlmProviders, TokenCount openai_count_tokens_handler: Final = OpenAICountTokensHandler() -def _workload_identity_config(litellm_params: Mapping[str, object]) -> OpenAIWorkloadIdentityConfig | None: +def _workload_identity_config( + litellm_params: Mapping[str, object], +) -> OpenAIWorkloadIdentityConfig | OpenAIError | None: api_base: Final = litellm_params.get("api_base") - return resolve_openai_workload_identity_config( - api_key=None, - api_base=api_base if isinstance(api_base, str) else None, - litellm_params=litellm_params, - ) + try: + return resolve_openai_workload_identity_config( + api_key=None, + api_base=api_base if isinstance(api_base, str) else None, + litellm_params=litellm_params, + ) + except OpenAIError as e: + return e class OpenAITokenCounter(BaseTokenCounter): @@ -126,7 +132,18 @@ class OpenAITokenCounter(BaseTokenCounter): credentials: Final = get_openai_credentials( api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key") ) - credential: Final = credentials.api_key or _workload_identity_config(litellm_params) + credential: Final = openai_static_api_key(credentials.api_key) or _workload_identity_config(litellm_params) + + if isinstance(credential, OpenAIError): + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="openai_api", + error=True, + error_message=credential.message, + status_code=credential.status_code, + ) if credential is None: return TokenCountResponse( diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 733d7930144..48a44575989 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional, cast import httpx @@ -97,6 +98,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): max_retries=max_retries, logging_obj=logging_obj, shared_session=shared_session, + litellm_params=litellm_params, ) openai_client: Final[OpenAI] = self._get_openai_client( @@ -106,6 +108,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, max_retries=max_retries, client=client, + litellm_params=litellm_params, ) ## LOGGING @@ -159,6 +162,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): client=None, max_retries=None, shared_session: Optional["ClientSession"] = None, + litellm_params: Mapping[str, object] | None = None, ): try: openai_aclient: Final[AsyncOpenAI] = self._get_openai_client( @@ -169,6 +173,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): max_retries=max_retries, client=client, shared_session=shared_session, + litellm_params=litellm_params, ) ## LOGGING diff --git a/litellm/llms/openai/vector_store_files/transformation.py b/litellm/llms/openai/vector_store_files/transformation.py index d5f15d639d6..7f65ade224e 100644 --- a/litellm/llms/openai/vector_store_files/transformation.py +++ b/litellm/llms/openai/vector_store_files/transformation.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) +from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_store_files import ( @@ -70,11 +71,13 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): headers: dict[str, str], litellm_params: GenericLiteLLMParams | None, ) -> dict[str, str]: - litellm_params = litellm_params or GenericLiteLLMParams() - api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + params: Final = litellm_params or GenericLiteLLMParams() + bearer_token: Final = resolve_openai_bearer_token( + api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True) + ) headers.update( { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json", } ) diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 57fe5b04838..46105a66be3 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -5,6 +5,7 @@ import httpx import litellm from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -48,11 +49,13 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): } def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict: - litellm_params = litellm_params or GenericLiteLLMParams() - api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + params: Final = litellm_params or GenericLiteLLMParams() + bearer_token: Final = resolve_openai_bearer_token( + api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True) + ) headers.update( { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json", } ) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 6e28695a96f..d422e51f649 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -8,11 +8,10 @@ from urllib.parse import quote import httpx from httpx._types import FileContent, FileTypes, RequestFiles -import litellm from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils -from litellm.secret_managers.main import get_secret_str +from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.types.llms.openai import CreateVideoRequest from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import ( @@ -78,16 +77,13 @@ class OpenAIVideoConfig(BaseVideoConfig): api_key: str | None = None, litellm_params: GenericLiteLLMParams | None = None, ) -> dict: - # Use api_key from litellm_params if available, otherwise fall back to other sources - if litellm_params and litellm_params.api_key: - api_key = api_key or litellm_params.api_key - - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - headers.update( - { - "Authorization": f"Bearer {api_key}", - } + params: Final = litellm_params or GenericLiteLLMParams() + bearer_token: Final = resolve_openai_bearer_token( + api_key=api_key or params.api_key, + api_base=params.api_base, + litellm_params=params.model_dump(exclude_none=True), ) + headers.update({"Authorization": f"Bearer {bearer_token}"}) return headers def get_complete_url( diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py index e2971a2c100..5952ae3b3ef 100644 --- a/litellm/llms/openai/workload_identity.py +++ b/litellm/llms/openai/workload_identity.py @@ -6,10 +6,22 @@ from functools import lru_cache from typing import TYPE_CHECKING, Final from urllib.parse import urlparse -import litellm -from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str +import httpx +from openai import DEFAULT_MAX_RETRIES as OPENAI_SDK_DEFAULT_MAX_RETRIES +from openai import NOT_GIVEN, AsyncOpenAI, NotGiven, OpenAI -from .common_utils import OpenAIError, is_openai_backed_api_base +import litellm +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE +from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str +from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS + +from .common_utils import ( + BaseOpenAILLM, + OpenAIAsyncHTTPClient, + OpenAIError, + OpenAIHTTPClient, + is_openai_backed_api_base, +) if TYPE_CHECKING: from collections.abc import Callable @@ -17,6 +29,11 @@ if TYPE_CHECKING: from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth OPENAI_WIF_CLIENT_ID: Final = "litellm" +_IDENTITY_PARAM_ENV_NAMES: Final = ( + ("openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"), + ("openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID"), + ("openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE"), +) _SDK_UPGRADE_MESSAGE: Final = ( "OpenAI workload identity federation requires openai>=2.32.0. " "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / " @@ -51,17 +68,25 @@ def resolve_openai_workload_identity_config( ) if static_api_key is not None: return None - effective_api_base: Final = ( - api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") - ) - if not _targets_openai_api(effective_api_base): + return _configured_openai_workload_identity(api_base=api_base, litellm_params=litellm_params) + + +def _effective_openai_api_base(api_base: str | None) -> str | None: + return api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") + + +def _configured_openai_workload_identity( + api_base: str | None, + litellm_params: Mapping[str, object] | None, +) -> OpenAIWorkloadIdentityConfig | None: + if not _targets_openai_api(_effective_openai_api_base(api_base)): return None - identity_provider_id: Final = _config_value( - litellm_params, "openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID" + identity_provider_id, service_account_id, token_file = ( + _config_value(litellm_params, param_key, env_name) for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES ) - service_account_id: Final = _config_value(litellm_params, "openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID") - token_file: Final = _config_value(litellm_params, "openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE") if identity_provider_id is None or service_account_id is None or token_file is None: + if _carries_deployment_identity(litellm_params): + raise OpenAIError(status_code=401, message=_incomplete_identity_message(litellm_params)) return None return OpenAIWorkloadIdentityConfig( identity_provider_id=identity_provider_id, @@ -81,11 +106,200 @@ async def get_workload_identity_bearer_token_for_api_base(api_base: str) -> str return await _workload_identity_auth(config).get_token_async() +def openai_static_api_key(api_key: str | None) -> str | None: + return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + + +def resolve_openai_bearer_token( + api_key: str | None, + api_base: str | None, + litellm_params: Mapping[str, object] | None, +) -> str | None: + static_api_key: Final = openai_static_api_key(api_key) + workload_identity_config: Final = resolve_openai_workload_identity_config( + api_key=static_api_key, api_base=api_base, litellm_params=litellm_params + ) + if workload_identity_config is None: + return static_api_key + return get_workload_identity_bearer_token(workload_identity_config) + + +async def resolve_openai_bearer_token_async( + static_api_key: str | None, + api_base: str | None, + litellm_params: Mapping[str, object] | None, +) -> str | None: + workload_identity_config: Final = resolve_openai_workload_identity_config( + api_key=static_api_key, api_base=api_base, litellm_params=litellm_params + ) + if workload_identity_config is None: + return static_api_key + return await _workload_identity_auth(workload_identity_config).get_token_async() + + +def _carries_deployment_identity(litellm_params: Mapping[str, object] | None) -> bool: + if litellm_params is None: + return False + return any(_param_str(litellm_params, key) is not None for key in OPENAI_WIF_KWARGS_KEYS) + + +def _incomplete_identity_message(litellm_params: Mapping[str, object] | None) -> str: + missing: Final = ", ".join( + param_key + for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES + if _config_value(litellm_params, param_key, env_name) is None + ) + return ( + f"OpenAI workload identity federation on this deployment is missing {missing}. " + "Set openai_identity_provider_id, openai_service_account_id, and openai_identity_token_file " + "on the deployment, or the OPENAI_IDENTITY_PROVIDER_ID / OPENAI_SERVICE_ACCOUNT_ID / " + "OPENAI_IDENTITY_TOKEN_FILE env vars they fall back to." + ) + + +def _param_str(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) and value else None + + def _config_value(litellm_params: Mapping[str, object] | None, param_key: str, env_name: str) -> str | None: - param_value: Final = litellm_params.get(param_key) if litellm_params is not None else None - if isinstance(param_value, str) and param_value: - return param_value - return normalize_nonempty_secret_str(get_secret_str(env_name)) + param_value: Final = _param_str(litellm_params, param_key) if litellm_params is not None else None + return param_value or normalize_nonempty_secret_str(get_secret_str(env_name)) + + +def build_openai_client( + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + max_retries: int | None = None, + organization: str | None = None, + litellm_params: Mapping[str, object] | None = None, + static_key_http_client_factory: Callable[[], httpx.Client | None] | None = None, +) -> OpenAI: + workload_identity_config: Final = resolve_openai_workload_identity_config( + api_key=api_key, api_base=api_base, litellm_params=litellm_params + ) + retries: Final = _max_retries_or_sdk_default(max_retries) + if workload_identity_config is None: + static_key_http_client: Final = ( + None if static_key_http_client_factory is None else static_key_http_client_factory() + ) + return OpenAI( + api_key=openai_static_api_key(api_key), + base_url=api_base, + timeout=timeout, + max_retries=retries, + organization=organization, + http_client=static_key_http_client or OpenAIHTTPClient(), + ) + cache_params: Final = _client_cache_params( + is_async=False, + workload_identity_config=workload_identity_config, + api_base=api_base, + timeout=timeout, + max_retries=retries, + organization=organization, + ) + cached: Final = BaseOpenAILLM.get_cached_openai_client( + client_initialization_params=cache_params, client_type="openai" + ) + if isinstance(cached, OpenAI): + return cached + process_http_client: Final = BaseOpenAILLM.get_sync_http_client() + client: Final = OpenAI( + workload_identity=workload_identity_config.to_sdk_workload_identity(), + base_url=api_base, + timeout=timeout, + max_retries=retries, + organization=organization, + http_client=process_http_client, + ) + BaseOpenAILLM.set_cached_openai_client( + openai_client=client, + client_type="openai", + client_initialization_params=cache_params, + litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client), + ) + return client + + +def build_async_openai_client( + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + max_retries: int | None = None, + organization: str | None = None, + litellm_params: Mapping[str, object] | None = None, + static_key_http_client_factory: Callable[[], httpx.AsyncClient | None] | None = None, +) -> AsyncOpenAI: + workload_identity_config: Final = resolve_openai_workload_identity_config( + api_key=api_key, api_base=api_base, litellm_params=litellm_params + ) + retries: Final = _max_retries_or_sdk_default(max_retries) + if workload_identity_config is None: + static_key_http_client: Final = ( + None if static_key_http_client_factory is None else static_key_http_client_factory() + ) + return AsyncOpenAI( + api_key=openai_static_api_key(api_key), + base_url=api_base, + timeout=timeout, + max_retries=retries, + organization=organization, + http_client=static_key_http_client or OpenAIAsyncHTTPClient(), + ) + cache_params: Final = _client_cache_params( + is_async=True, + workload_identity_config=workload_identity_config, + api_base=api_base, + timeout=timeout, + max_retries=retries, + organization=organization, + ) + cached: Final = BaseOpenAILLM.get_cached_openai_client( + client_initialization_params=cache_params, client_type="openai" + ) + if isinstance(cached, AsyncOpenAI): + return cached + process_http_client: Final = BaseOpenAILLM.get_async_http_client() + client: Final = AsyncOpenAI( + workload_identity=workload_identity_config.to_sdk_workload_identity(), + base_url=api_base, + timeout=timeout, + max_retries=retries, + organization=organization, + http_client=process_http_client, + ) + BaseOpenAILLM.set_cached_openai_client( + openai_client=client, + client_type="openai", + client_initialization_params=cache_params, + litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client), + ) + return client + + +def _client_cache_params( + is_async: bool, + workload_identity_config: OpenAIWorkloadIdentityConfig, + api_base: str | None, + timeout: float | httpx.Timeout | None | NotGiven, + max_retries: int, + organization: str | None, +) -> dict[str, object]: + return { + "api_key": None, + "is_async": is_async, + "workload_identity_config": workload_identity_config, + "api_base": api_base, + "timeout": timeout, + "max_retries": max_retries, + "organization": organization, + } + + +def _max_retries_or_sdk_default(max_retries: int | None) -> int: + return OPENAI_SDK_DEFAULT_MAX_RETRIES if max_retries is None else max_retries def _targets_openai_api(api_base: str | None) -> bool: @@ -95,7 +309,7 @@ def _targets_openai_api(api_base: str | None) -> bool: return parsed.scheme == "https" and is_openai_backed_api_base(api_base) -@lru_cache(maxsize=16) +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth: sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth() return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity()) diff --git a/litellm/main.py b/litellm/main.py index 3646b48b8cc..23cd3c40250 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -255,6 +255,7 @@ from .llms.openai.completion.handler import OpenAITextCompletion from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler from .llms.openai.openai import OpenAIChatCompletion from .llms.openai.transcriptions.handler import OpenAIAudioTranscription +from .llms.openai.workload_identity import build_openai_client from .llms.openai_like.chat.handler import OpenAILikeChatHandler from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig @@ -7866,17 +7867,14 @@ def adapter_completion(*, adapter_id: str, **kwargs) -> BaseModel | AdapterCompl def moderation(input: str, model: str | None = None, api_key: str | None = None, **kwargs) -> OpenAIModerationResponse: - from litellm.llms.openai.common_utils import OpenAIHTTPClient - - # only supports open ai for now - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - - # Extract api_base from kwargs + static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") api_base: Final = kwargs.get("api_base", None) - openai_client = kwargs.get("client", None) - if openai_client is None: - openai_client = openai.OpenAI(api_key=api_key, base_url=api_base, http_client=OpenAIHTTPClient()) + openai_client: Final = kwargs.get("client", None) or build_openai_client( + api_key=static_api_key, + api_base=api_base, + litellm_params=kwargs, + ) if model is not None: response = openai_client.moderations.create(input=input, model=model) @@ -7899,8 +7897,7 @@ async def amoderation( ) -> OpenAIModerationResponse: from openai import AsyncOpenAI - # only supports open ai for now - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") optional_params: Final = GenericLiteLLMParams.model_validate(kwargs) litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None) _dynamic_api_base = None @@ -7926,8 +7923,9 @@ async def amoderation( # _get_openai_client maintains in-memory caching logic for OpenAI clients _openai_client: AsyncOpenAI = openai_chat_completions.get_openai_client( is_async=True, - api_key=api_key, + api_key=static_api_key, api_base=optional_params.api_base or _dynamic_api_base, + litellm_params=optional_params.model_dump(exclude_none=True), ) else: _openai_client = openai_client @@ -7947,7 +7945,7 @@ async def amoderation( moderation_request: Final = {"input": input, "model": model} litellm_logging_obj.pre_call( input=input, - api_key=api_key, + api_key=static_api_key, additional_args={ "complete_input_dict": moderation_request, "api_base": str(_openai_client.base_url), @@ -8180,9 +8178,7 @@ def transcription( or get_secret("OPENAI_ORGANIZATION") or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) - # set API KEY - - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") + static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") response = openai_audio_transcriptions.audio_transcriptions( model=model, audio_file=file, @@ -8194,7 +8190,7 @@ def transcription( logging_obj=litellm_logging_obj, max_retries=max_retries, api_base=api_base, - api_key=api_key, + api_key=static_api_key, provider_config=provider_config, litellm_params=litellm_params_dict, shared_session=shared_session, @@ -8516,6 +8512,7 @@ def speech( client=client, # pass AsyncOpenAI, OpenAI client aspeech=aspeech, shared_session=shared_session, + litellm_params=litellm_params_dict, ) elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS: # Check if this is Azure Speech Service (Cognitive Services TTS) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 98bb0985f4a..94cf7ff4dbe 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -551,6 +551,29 @@ def _iter_fallback_targets(value: object, depth: int) -> Iterator[str | Mapping[ yield from _iter_fallback_targets(target_list, depth + 1) +def _reject_client_chosen_federation(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: + reject_server_owned_wif_params(litellm_params) + if not manages_deployments: + reject_federated_credential_reference(litellm_params) + + +def _iter_user_config_litellm_params(user_config: object) -> Iterator[Mapping[str, object]]: + config: Final = _coerce_metadata_to_dict(user_config) + if config is None: + return + if (defaults := _coerce_metadata_to_dict(config.get("default_litellm_params"))) is not None: + yield defaults + model_list: Final = config.get("model_list") + if not isinstance(model_list, list): + return + for deployment in model_list: + if ( + isinstance(deployment, Mapping) + and (params := _coerce_metadata_to_dict(deployment.get("litellm_params"))) is not None + ): + yield params + + def iter_request_fallback_targets(request_body: Mapping[str, object]) -> Iterator[str | Mapping[str, object]]: for value in _iter_fallback_field_values(request_body): yield from _iter_fallback_targets(value, 0) @@ -658,11 +681,11 @@ def is_request_body_safe( _reject_url_valued_fallback_target(target_model) elif isinstance(target, str): _reject_url_valued_fallback_target(target) + for user_config_params in _iter_user_config_litellm_params(request_body.get("user_config")): + _reject_client_chosen_federation(user_config_params, manages_deployments=manages_deployments) litellm_params: Final = _coerce_metadata_to_dict(request_body.get("litellm_params")) if litellm_params is not None: - reject_server_owned_wif_params(litellm_params) - if not manages_deployments: - reject_federated_credential_reference(litellm_params) + _reject_client_chosen_federation(litellm_params, manages_deployments=manages_deployments) litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata")) if litellm_params_metadata is not None: _check_banned_params( diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 8bf39b7a17a..ead1cd8e2ca 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -17,6 +17,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest from litellm.litellm_core_utils.hidden_params import get_hidden_params, set_hidden_param +from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit @@ -171,6 +172,8 @@ def _raise_not_found_when_openai_fallback_unservable( return if data.get("api_key") or litellm.api_key or litellm.openai_key or os.getenv("OPENAI_API_KEY"): return + if resolve_openai_workload_identity_config(api_key=None, api_base=None, litellm_params=data) is not None: + return raise ProxyException( message=not_found_message, type="invalid_request_error", diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py index 74bba5c33da..fe526b72ac3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py @@ -5,6 +5,7 @@ from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) from litellm.types.guardrails import SupportedGuardrailIntegrations +from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS if TYPE_CHECKING: from litellm.types.guardrails import Guardrail, LitellmParams @@ -19,6 +20,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" openai_moderation_guardrail: Final = OpenAIModerationGuardrail( guardrail_name=guardrail_name, + workload_identity_params=litellm_params.model_dump(include=set(OPENAI_WIF_KWARGS_KEYS), exclude_none=True), **{ **litellm_params.model_dump(exclude_none=True), "api_key": litellm_params.api_key, diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index b246b125909..33884fda4ee 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -3,6 +3,8 @@ OpenAI Moderation Guardrail Integration for LiteLLM """ +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi import HTTPException @@ -17,7 +19,14 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.openai.workload_identity import ( + OpenAIWorkloadIdentityConfig, + resolve_openai_bearer_token_async, + resolve_openai_workload_identity_config, +) from litellm.proxy.common_utils.callback_utils import add_guardrail_scan_id +from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations from litellm.types.utils import ( GenericGuardrailAPIInputs, @@ -32,6 +41,21 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +def _config_secret(value: object) -> object: + return get_secret_str(value) if isinstance(value, str) and value.startswith("os.environ/") else value + + +def _guardrail_workload_identity( + api_base: str, workload_identity_params: Mapping[str, object] +) -> OpenAIWorkloadIdentityConfig | None: + try: + return resolve_openai_workload_identity_config( + api_key=None, api_base=api_base, litellm_params=workload_identity_params + ) + except OpenAIError as incomplete_identity: + raise ValueError(f"OpenAI Moderation: {incomplete_identity.message}") from incomplete_identity + + class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): """ LiteLLM Built-in Guardrail for OpenAI Content Moderation. @@ -55,6 +79,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): model: Literal["omni-moderation-latest", "text-moderation-latest"] | None = None, streaming_end_of_stream_only: bool | None = None, streaming_sampling_rate: int | None = None, + workload_identity_params: Mapping[str, object] | None = None, **kwargs, ): """Initialize OpenAI Moderation guardrail handler.""" @@ -78,7 +103,11 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): ) self.streaming_sampling_rate: int = 5 if streaming_sampling_rate is None else streaming_sampling_rate - if not self.api_key: + self.workload_identity_params: Mapping[str, object] = MappingProxyType( + {key: _config_secret(value) for key, value in (workload_identity_params or {}).items()} + ) + + if not self.api_key and _guardrail_workload_identity(self.api_base, self.workload_identity_params) is None: raise ValueError( "OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration." ) @@ -109,10 +138,13 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): verbose_proxy_logger.debug("OpenAI Moderation guard request: %s", request_body) + bearer_token: Final = await resolve_openai_bearer_token_async( + static_api_key=self.api_key, api_base=self.api_base, litellm_params=self.workload_identity_params + ) response: Final = await self.async_handler.post( url=f"{self.api_base}/moderations", headers={ - "Authorization": f"Bearer {self.api_key}", + "Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json", }, json=request_body, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index aeec0bf67c0..4acce4de586 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -43,6 +43,7 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime from ..llms.custom_httpx.http_handler import realtime_ssl_for_url from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config from ..llms.vertex_ai.vertex_llm_base import VertexBase @@ -130,6 +131,17 @@ def _get_realtime_http_provider_config( return provider_config, resolved_api_base.rstrip("/"), resolved_api_key +async def _realtime_http_bearer_token( + custom_llm_provider: str, api_key: str, api_base: str, litellm_params: GenericLiteLLMParams +) -> str: + if custom_llm_provider != LlmProviders.OPENAI.value: + return api_key + bearer_token: Final = await resolve_openai_bearer_token_async( + static_api_key=api_key or None, api_base=api_base, litellm_params=litellm_params.model_dump(exclude_none=True) + ) + return bearer_token or "" + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, @@ -180,7 +192,9 @@ async def acreate_realtime_client_secret( request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, - api_key=resolved_api_key, + api_key=await _realtime_http_bearer_token( + custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params + ), request_data=request_data, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, @@ -253,7 +267,9 @@ async def acreate_realtime_transcription_session( request_data["input_audio_transcription"]["model"] = model_name return await base_llm_http_handler.async_realtime_transcription_session_handler( api_base=resolved_api_base, - api_key=resolved_api_key, + api_key=await _realtime_http_bearer_token( + custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params + ), request_data=request_data, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, @@ -418,17 +434,21 @@ async def _alive_session( api_base: str | None, api_key: str | None, litellm_metadata: Mapping[str, object], + litellm_params: Mapping[str, object], ) -> None: rejection: Final = _live_session_rejection(custom_llm_provider, litellm_metadata) if rejection is not None: _raise_live_session_rejection(rejection) + resolved_api_base: Final = api_base or litellm.api_base or "https://api.openai.com/" await openai_live_sessions.async_live_session( model=model, websocket=websocket, logging_obj=litellm_logging_obj, session_start=session_start, - api_base=api_base or litellm.api_base or "https://api.openai.com/", - api_key=api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY"), + api_base=resolved_api_base, + api_key=await resolve_openai_bearer_token_async( + static_api_key=openai_static_api_key(api_key), api_base=resolved_api_base, litellm_params=litellm_params + ), ) @@ -497,6 +517,7 @@ async def _arealtime( api_base=dynamic_api_base or litellm_params.api_base, api_key=dynamic_api_key, litellm_metadata=_build_litellm_metadata(kwargs), + litellm_params=kwargs, ) return @@ -558,8 +579,9 @@ async def _arealtime( ) elif _custom_llm_provider == "openai": api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/" - # set API KEY - api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + api_key = await resolve_openai_bearer_token_async( + static_api_key=openai_static_api_key(dynamic_api_key), api_base=api_base, litellm_params=kwargs + ) await openai_realtime.async_realtime( model=model, @@ -774,9 +796,18 @@ async def realtime_health_check( str | None, api_version or resolved_params.get("api_version") ) url: str | None = None + health_check_api_key: Final = ( + await resolve_openai_bearer_token_async( + static_api_key=openai_static_api_key(resolved_api_key), + api_base=resolved_api_base, + litellm_params=resolved_params, + ) + if custom_llm_provider == LlmProviders.OPENAI.value + else resolved_api_key + ) auth_headers: Final = _realtime_health_check_auth_headers( custom_llm_provider=custom_llm_provider, - api_key=resolved_api_key, + api_key=health_check_api_key, model_params=resolved_params, ) if custom_llm_provider == "azure": diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index c91f504eefa..01f151ef4d0 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -1,4 +1,6 @@ +import asyncio from collections.abc import Mapping +from functools import partial from typing import Final import httpx @@ -75,21 +77,24 @@ async def asystemone( assets: DecisionsAssetEntries | None = None, **kwargs: object, # kwargs-ok: litellm_params forwarded by @client ) -> DecisionsResponse: - call: Final = _prepare_systemone_call( - model=model, - state=state, - questions=questions, - images=images, - assets=assets, - safety_identifier=safety_identifier, - audio=audio, - videos=videos, - api_key=api_key, - api_base=api_base, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - kwargs=kwargs, + call: Final = await asyncio.to_thread( + partial( + _prepare_systemone_call, + model=model, + state=state, + questions=questions, + images=images, + assets=assets, + safety_identifier=safety_identifier, + audio=audio, + videos=videos, + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) ) return _systemone_response(await asend(call)) diff --git a/litellm/types/router.py b/litellm/types/router.py index 8f3ba7c0a8d..96add8bb2a1 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1402,8 +1402,9 @@ def reject_server_owned_wif_params(body: Mapping[str, object]) -> None: This lives here rather than under ``litellm.proxy`` so the router can call it on a post-authentication merge without core importing from the proxy package. """ + submitted_names: Final = frozenset(key.removesuffix("[]") for key in body) for param in _server_owned_wif_litellm_params: - if param in body: + if param in submitted_names: raise ValueError( f"Rejected Request: {param} is a server-owned workload identity federation or OAuth token exchange " "parameter " diff --git a/tests/integration/compatibility/test_audio_transcriptions_wire.py b/tests/integration/compatibility/test_audio_transcriptions_wire.py index 02a09bce5fd..8f9a83e9912 100644 --- a/tests/integration/compatibility/test_audio_transcriptions_wire.py +++ b/tests/integration/compatibility/test_audio_transcriptions_wire.py @@ -604,6 +604,31 @@ def test_audio_transcription_consumes_bracketed_form_metadata_tags(gateway: Gate ) == (tag,), rows +@pytest.mark.parametrize( + "identity_param", ("openai_identity_provider_id", "openai_service_account_id", "openai_identity_token_file") +) +def test_audio_transcription_refuses_a_bracketed_identity_form_key_before_the_upstream( + gateway: Gateway, identity_param: str +) -> None: + wav: Final = _WAV_BYTES + uuid.uuid4().bytes + + def respond(request: Request) -> Reply: + return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + response: Final = gateway.request_multipart( + "/v1/audio/transcriptions", + {"model": model, "response_format": "json", f"{identity_param}[]": "caller-chosen-" + uuid.uuid4().hex}, + {"file": ("a.wav", wav, _WAV_CONTENT_TYPE)}, + ) + assert response.status_code == 401, response.text + assert identity_param in response.text, response.text + assert wire.drain() == () + + def test_audio_transcription_preserves_the_client_file_content_type(gateway: Gateway) -> None: pytest.skip( "BUG: the proxy discards the client's file part content type and re-derives it from the filename, " diff --git a/tests/integration/routing/test_user_config_routing.py b/tests/integration/routing/test_user_config_routing.py index 1c50a78201b..af3df8a38ff 100644 --- a/tests/integration/routing/test_user_config_routing.py +++ b/tests/integration/routing/test_user_config_routing.py @@ -3,8 +3,10 @@ import uuid from pathlib import Path from typing import Final +import pytest + from integration._support.client import Gateway -from integration._support.process import owned_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.wire import Reply, Request, wire_server USER_KEY: Final = "sk-user-supplied-" + uuid.uuid4().hex @@ -30,18 +32,13 @@ def _completion(request: Request) -> Reply: ) +def _deployment_params(upstream_url: str) -> dict[str, object]: + return {"model": "openai/gpt-4.1-mini", "api_base": upstream_url + "/v1", "api_key": USER_KEY} + + def _user_config(upstream_url: str) -> dict[str, object]: return { - "model_list": [ - { - "model_name": "user-config-deployment", - "litellm_params": { - "model": "openai/gpt-4.1-mini", - "api_base": upstream_url + "/v1", - "api_key": USER_KEY, - }, - } - ], + "model_list": [{"model_name": "user-config-deployment", "litellm_params": _deployment_params(upstream_url)}], "num_retries": 0, } @@ -95,6 +92,54 @@ def test_user_config_routes_to_the_user_supplied_deployment_when_opted_in(gatewa assert json.loads(completions[0].body)["model"] == "gpt-4.1-mini" +def _identity_in_deployment(upstream_url: str, identity_param: str) -> dict[str, object]: + return { + "model_list": [ + { + "model_name": "user-config-deployment", + "litellm_params": {**_deployment_params(upstream_url), identity_param: "caller-chosen"}, + } + ], + "num_retries": 0, + } + + +def _identity_in_defaults(upstream_url: str, identity_param: str) -> dict[str, object]: + return {**_user_config(upstream_url), "default_litellm_params": {identity_param: "caller-chosen"}} + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +def test_user_config_cannot_carry_a_server_owned_identity_param_even_when_opted_in( + gateway: Gateway, tmp_path: Path +) -> None: + cases: Final = ( + (_identity_in_deployment, "openai_identity_provider_id"), + (_identity_in_defaults, "openai_service_account_id"), + ) + with wire_server(_completion) as upstream: + config: Final = _opt_in_config(tmp_path, upstream.url) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + responses: Final = tuple( + ( + identity_param, + candidate.request( + "POST", + "/v1/chat/completions", + { + "model": "user-config-deployment", + "messages": [{"role": "user", "content": "user config identity"}], + "user_config": placement(upstream.url, identity_param), + }, + ), + ) + for placement, identity_param in cases + ) + for identity_param, response in responses: + assert response.status_code == 401, response.text + assert identity_param in response.text, response.text + assert tuple(request for request in upstream.drain() if request.target == "/v1/chat/completions") == () + + def test_user_config_is_rejected_without_the_opt_in(gateway: Gateway) -> None: with wire_server(_completion) as upstream: response: Final = gateway.request("POST", "/v1/chat/completions", _request_body(upstream.url)) diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 293c38e9f20..912d4cfe84c 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -1983,6 +1983,20 @@ def test_total_cost_bills_cached_tokens_per_line_at_the_batch_cached_rate(): assert result.cost == pytest.approx(long_line + short_line) +def test_extract_credentials_keeps_openai_workload_identity_params(): + params = { + "openai_identity_provider_id": "idp_deployment", + "openai_service_account_id": "user-deployment", + "openai_identity_token_file": "/var/run/secrets/token", + "model": "gpt-4o", + } + assert bu._extract_file_access_credentials(params) == { + "openai_identity_provider_id": "idp_deployment", + "openai_service_account_id": "user-deployment", + "openai_identity_token_file": "/var/run/secrets/token", + } + + # --------------------------------------------------------------------------- # # batch_cost_is_final # --------------------------------------------------------------------------- # diff --git a/tests/unit/llms/openai/completion/test_handler.py b/tests/unit/llms/openai/completion/test_handler.py index b4b224f0977..6a7302ec122 100644 --- a/tests/unit/llms/openai/completion/test_handler.py +++ b/tests/unit/llms/openai/completion/test_handler.py @@ -106,7 +106,7 @@ async def test_acompletion_uses_optimized_http_client(): with patch.object(BaseOpenAILLM, "_get_async_http_client", return_value=mock_http_client) as mock_get_client: with patch( - "litellm.llms.openai.completion.handler.AsyncOpenAI", + "litellm.llms.openai.workload_identity.AsyncOpenAI", return_value=mock_async_openai, ) as mock_openai_class: handler = OpenAITextCompletion() diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index 3c7dff34296..831df30c98a 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -442,15 +442,26 @@ def _sdk_api_client( ) if api == "image_variations": variation_factory: Final = OpenAIImageVariationsHandler() - params: Final = { - "api_key": "transport-only", - "base_url": kwargs["api_base"], - "timeout": timeout, - } return ( - variation_factory.get_async_client(client=client, init_client_params=params) + variation_factory.get_async_client( + client=client, + api_key="transport-only", + api_base="https://sdk-default.example/v1", + timeout=timeout, + max_retries=0, + organization=None, + litellm_params={}, + ) if is_async - else variation_factory.get_sync_client(client=client, init_client_params=params) + else variation_factory.get_sync_client( + client=client, + api_key="transport-only", + api_base="https://sdk-default.example/v1", + timeout=timeout, + max_retries=0, + organization=None, + litellm_params={}, + ) ) if api == "azure_gateway": return BaseAzureLLM()._init_azure_client_for_cloudflare_ai_gateway( diff --git a/tests/unit/llms/openai/test_openai_workload_identity.py b/tests/unit/llms/openai/test_openai_workload_identity.py index 2236de1eb5f..1a1a4ca7047 100644 --- a/tests/unit/llms/openai/test_openai_workload_identity.py +++ b/tests/unit/llms/openai/test_openai_workload_identity.py @@ -1,5 +1,9 @@ +import asyncio +import io import json import sys +import threading +from collections.abc import Awaitable, Callable from functools import partial from pathlib import Path from typing import Final @@ -7,21 +11,36 @@ from typing import Final import httpx import pytest import respx -from openai import AsyncOpenAI, OpenAI +from openai import DEFAULT_MAX_RETRIES as OPENAI_SDK_DEFAULT_MAX_RETRIES +from openai import APIStatusError, AsyncOpenAI, OpenAI import litellm +from litellm.containers import alist_container_files, aretrieve_container, list_container_files +from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIError +from litellm.llms.openai.containers.transformation import OpenAIContainerConfig +from litellm.llms.openai.evals.transformation import OpenAIEvalsConfig +from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.llms.openai.openai import OpenAIChatCompletion +from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openai.vector_store_files.transformation import OpenAIVectorStoreFilesConfig +from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig +from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.llms.openai.workload_identity import ( OpenAIWorkloadIdentityConfig, _workload_identity_auth, + build_async_openai_client, + build_openai_client, get_workload_identity_bearer_token, + resolve_openai_bearer_token, resolve_openai_workload_identity_config, ) from litellm.types.router import GenericLiteLLMParams +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem +from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS TOKEN_EXCHANGE_URL: Final = "https://auth.openai.com/oauth/token" CHAT_COMPLETIONS_URL: Final = "https://api.openai.com/v1/chat/completions" @@ -190,6 +209,25 @@ class TestTokenExchange: assert first == second == "exchanged-bearer-token" assert route.call_count == 1 + @respx.mock + def test_more_identities_than_the_old_cache_ceiling_stay_cached( + self, wif_env: OpenAIWorkloadIdentityConfig + ) -> None: + route: Final = mock_token_exchange() + identities: Final = tuple( + OpenAIWorkloadIdentityConfig( + identity_provider_id=wif_env.identity_provider_id, + service_account_id=f"user-{index}", + token_file=wif_env.token_file, + ) + for index in range(24) + ) + for identity in identities: + assert get_workload_identity_bearer_token(identity) == "exchanged-bearer-token" + for identity in identities: + assert get_workload_identity_bearer_token(identity) == "exchanged-bearer-token" + assert route.call_count == len(identities) + def test_old_sdk_raises_upgrade_error( self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -410,9 +448,60 @@ class TestResolveConfigFromDeployment: ) assert config == wif_env - def test_partial_litellm_params_without_env_disable(self, deployment_wif: dict[str, str]) -> None: - partial: Final = {key: value for key, value in deployment_wif.items() if key != "openai_identity_token_file"} - assert resolve_openai_workload_identity_config(api_key=None, api_base=None, litellm_params=partial) is None + @pytest.mark.parametrize("missing_key", sorted(OPENAI_WIF_KWARGS_KEYS)) + def test_partial_litellm_params_without_env_raise_naming_the_missing_field( + self, deployment_wif: dict[str, str], missing_key: str + ) -> None: + subset: Final = {key: value for key, value in deployment_wif.items() if key != missing_key} + with pytest.raises(OpenAIError) as error: + resolve_openai_workload_identity_config(api_key=None, api_base=None, litellm_params=subset) + assert error.value.status_code == 401 + assert f"missing {missing_key}." in error.value.message + assert all(key not in error.value.message.split(".")[0] for key in subset) + + @respx.mock + @pytest.mark.parametrize( + "call", + [ + partial(litellm.completion, model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "hi"}]), + partial(litellm.image_generation, model="openai/gpt-image-2", prompt="a cat"), + ], + ids=["chat", "image"], + ) + def test_partial_deployment_identity_surfaces_as_an_auth_error_naming_the_gap( + self, deployment_wif: dict[str, str], call: Callable[..., object] + ) -> None: + subset: Final = {key: value for key, value in deployment_wif.items() if key != "openai_service_account_id"} + with pytest.raises(litellm.AuthenticationError) as error: + call(**subset) + assert error.value.status_code == 401 + assert "missing openai_service_account_id." in error.value.message + + def test_partial_litellm_params_fall_back_to_the_process_key( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + subset: Final = {key: value for key, value in deployment_wif.items() if key != "openai_identity_token_file"} + assert ( + resolve_openai_workload_identity_config(api_key="sk-from-env", api_base=None, litellm_params=subset) is None + ) + + def test_deployment_key_equal_to_the_process_key_still_wins( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-shared") + assert ( + resolve_openai_workload_identity_config( + api_key="sk-shared", api_base=None, litellm_params={**deployment_wif, "api_key": "sk-shared"} + ) + is None + ) + + def test_partial_litellm_params_yield_to_the_deployments_own_key(self, deployment_wif: dict[str, str]) -> None: + subset: Final = {key: value for key, value in deployment_wif.items() if key != "openai_identity_token_file"} + assert ( + resolve_openai_workload_identity_config(api_key="sk-static", api_base=None, litellm_params=subset) is None + ) def test_static_api_key_beats_litellm_params(self, deployment_wif: dict[str, str]) -> None: assert ( @@ -420,7 +509,7 @@ class TestResolveConfigFromDeployment: is None ) - def test_env_openai_api_key_beats_litellm_params( + def test_env_openai_api_key_beats_deployment_identity( self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") @@ -428,6 +517,16 @@ class TestResolveConfigFromDeployment: resolve_openai_workload_identity_config(api_key=None, api_base=None, litellm_params=deployment_wif) is None ) + @pytest.mark.parametrize("global_attr", ["api_key", "openai_key"]) + def test_litellm_module_key_beats_deployment_identity( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch, global_attr: str + ) -> None: + monkeypatch.setattr(litellm, global_attr, "sk-module-global") + assert ( + resolve_openai_bearer_token(api_key=None, api_base=None, litellm_params=deployment_wif) + == "sk-module-global" + ) + def test_foreign_api_base_disables_deployment_wif(self, deployment_wif: dict[str, str]) -> None: assert ( resolve_openai_workload_identity_config( @@ -468,6 +567,108 @@ class TestDeploymentClientConstruction: assert first is not second assert again is first + def test_builders_reuse_cached_client_per_identity(self, deployment_wif: dict[str, str]) -> None: + other_deployment: Final = {**deployment_wif, "openai_service_account_id": "user-other"} + first: Final = build_openai_client(api_key=None, api_base=None, litellm_params=deployment_wif) + again: Final = build_openai_client(api_key=None, api_base=None, litellm_params=dict(deployment_wif)) + other: Final = build_openai_client(api_key=None, api_base=None, litellm_params=other_deployment) + async_first: Final = build_async_openai_client(api_key=None, api_base=None, litellm_params=deployment_wif) + async_again: Final = build_async_openai_client(api_key=None, api_base=None, litellm_params=deployment_wif) + assert again is first + assert other is not first + assert async_again is async_first + assert isinstance(async_first, AsyncOpenAI) + + @pytest.mark.parametrize("builder", [build_openai_client, build_async_openai_client]) + def test_unset_max_retries_keep_the_sdk_default_and_explicit_ones_stick( + self, deployment_wif: dict[str, str], builder: Callable[..., OpenAI | AsyncOpenAI] + ) -> None: + assert ( + builder(api_key="sk-static", api_base=None, max_retries=None).max_retries == OPENAI_SDK_DEFAULT_MAX_RETRIES + ) + assert builder(api_key=None, api_base=None, max_retries=None, litellm_params=deployment_wif).max_retries == ( + OPENAI_SDK_DEFAULT_MAX_RETRIES + ) + assert builder(api_key=None, api_base=None, max_retries=0, litellm_params=deployment_wif).max_retries == 0 + + def test_builders_never_cache_static_key_clients(self) -> None: + first: Final = build_openai_client(api_key="sk-static", api_base=None, litellm_params=None) + again: Final = build_openai_client(api_key="sk-static", api_base=None, litellm_params=None) + assert again is not first + + @respx.mock + def test_shared_workload_identity_client_never_borrows_the_callers_transport( + self, deployment_wif: dict[str, str] + ) -> None: + mock_token_exchange("process-transport-bearer") + models_route: Final = respx.get(MODELS_URL).mock( + return_value=httpx.Response(200, json={"object": "list", "data": []}) + ) + caller_transport: Final = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(418))) + + federated: Final = build_openai_client( + api_key=None, + api_base=None, + litellm_params=deployment_wif, + static_key_http_client_factory=lambda: caller_transport, + ) + federated.models.list() + static: Final = build_openai_client( + api_key="sk-static", + api_base=None, + litellm_params=None, + static_key_http_client_factory=lambda: caller_transport, + ) + with pytest.raises(APIStatusError) as static_error: + static.models.list() + + assert models_route.calls.last.request.headers["Authorization"] == "Bearer process-transport-bearer" + assert static_error.value.status_code == 418 + + def test_federated_builders_never_call_the_static_key_client_factory(self, deployment_wif: dict[str, str]) -> None: + def factory_that_must_stay_cold() -> httpx.Client | None: + raise AssertionError("static-key http client built on the federated path") + + async def async_factory_that_must_stay_cold() -> httpx.AsyncClient | None: + raise AssertionError("static-key async http client built on the federated path") + + assert isinstance( + build_openai_client( + api_key=None, + api_base=None, + litellm_params=deployment_wif, + static_key_http_client_factory=factory_that_must_stay_cold, + ), + OpenAI, + ) + assert isinstance( + build_async_openai_client( + api_key=None, + api_base=None, + litellm_params=deployment_wif, + static_key_http_client_factory=async_factory_that_must_stay_cold, + ), + AsyncOpenAI, + ) + + @respx.mock + @pytest.mark.asyncio + async def test_shared_workload_identity_client_rides_the_process_session( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + mock_token_exchange("session-bearer") + session_models: Final = {"object": "list", "data": [{"id": "model-from-process-session", "object": "model"}]} + monkeypatch.setattr( + litellm, + "aclient_session", + httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=session_models))), + ) + + federated: Final = build_async_openai_client(api_key=None, api_base=None, litellm_params=deployment_wif) + listed: Final = await federated.models.list() + + assert [model.id for model in listed.data] == ["model-from-process-session"] + @respx.mock def test_completion_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: mock_token_exchange("deployment-bearer") @@ -565,6 +766,145 @@ class TestResponsesValidateEnvironmentFromDeployment: ) assert headers["Authorization"] == "Bearer sk-responses" + def test_static_key_equal_to_the_process_key_wins( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-shared") + assert ( + OpenAIResponsesAPIConfig().validate_environment( + headers={}, + model="gpt-4o-mini", + litellm_params=GenericLiteLLMParams(api_key="sk-shared", **deployment_wif), + )["Authorization"] + == "Bearer sk-shared" + ) + + +class TestResolveBearerToken: + @respx.mock + def test_keyless_deployment_mints_the_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + exchange: Final = mock_token_exchange("resolved-bearer") + assert ( + resolve_openai_bearer_token(api_key=None, api_base=None, litellm_params=deployment_wif) == "resolved-bearer" + ) + assert exchange.call_count == 1 + + @respx.mock + def test_deployment_key_wins_without_an_exchange(self, deployment_wif: dict[str, str]) -> None: + exchange: Final = mock_token_exchange() + assert ( + resolve_openai_bearer_token(api_key="sk-static", api_base=None, litellm_params=deployment_wif) + == "sk-static" + ) + assert exchange.call_count == 0 + + @respx.mock + def test_process_wide_key_beats_deployment_identity_without_an_exchange( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + exchange: Final = mock_token_exchange() + assert resolve_openai_bearer_token(api_key=None, api_base=None, litellm_params=deployment_wif) == "sk-from-env" + assert exchange.call_count == 0 + + def test_process_wide_key_serves_deployments_without_identity(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + assert ( + resolve_openai_bearer_token(api_key=None, api_base=None, litellm_params={"model": "gpt-5.6"}) + == "sk-from-env" + ) + + def test_nothing_configured_resolves_to_none(self, deployment_wif: dict[str, str]) -> None: + assert resolve_openai_bearer_token(api_key=None, api_base=None, litellm_params=None) is None + + +HeadersFromParams = Callable[[GenericLiteLLMParams], dict[str, str]] +HTTP_HANDLER_VALIDATORS: Final[tuple[tuple[str, HeadersFromParams], ...]] = ( + ( + "image_edit", + lambda params: OpenAIImageEditConfig().validate_environment( + headers={}, + model="gpt-image-2", + api_key=params.api_key, + litellm_params=dict(params), + api_base=params.api_base, + ), + ), + ("vector_stores", lambda params: OpenAIVectorStoreConfig().validate_environment(headers={}, litellm_params=params)), + ( + "vector_store_files", + lambda params: OpenAIVectorStoreFilesConfig().validate_environment(headers={}, litellm_params=params), + ), + ( + "videos", + lambda params: OpenAIVideoConfig().validate_environment( + headers={}, model="sora-2", api_key=params.api_key, litellm_params=params + ), + ), + ( + "containers", + lambda params: OpenAIContainerConfig().validate_environment( + headers={}, api_key=params.api_key, litellm_params=params + ), + ), + ("evals", lambda params: OpenAIEvalsConfig().validate_environment(headers={}, litellm_params=params)), +) + + +@pytest.mark.parametrize( + ("surface", "headers_from"), HTTP_HANDLER_VALIDATORS, ids=[name for name, _ in HTTP_HANDLER_VALIDATORS] +) +class TestHttpHandlersHonorDeploymentIdentity: + @respx.mock + def test_keyless_deployment_mints_the_exchanged_bearer( + self, deployment_wif: dict[str, str], surface: str, headers_from: HeadersFromParams + ) -> None: + exchange: Final = mock_token_exchange(f"{surface}-bearer") + headers: Final = headers_from(GenericLiteLLMParams(**deployment_wif)) + assert headers["Authorization"] == f"Bearer {surface}-bearer" + assert exchange.call_count == 1 + + @respx.mock + def test_deployment_key_wins_without_an_exchange( + self, deployment_wif: dict[str, str], surface: str, headers_from: HeadersFromParams + ) -> None: + exchange: Final = mock_token_exchange() + headers: Final = headers_from(GenericLiteLLMParams(api_key=f"sk-{surface}", **deployment_wif)) + assert headers["Authorization"] == f"Bearer sk-{surface}" + assert exchange.call_count == 0 + + @respx.mock + def test_process_wide_key_beats_deployment_identity_without_an_exchange( + self, + deployment_wif: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + surface: str, + headers_from: HeadersFromParams, + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + exchange: Final = mock_token_exchange() + headers: Final = headers_from(GenericLiteLLMParams(**deployment_wif)) + assert headers["Authorization"] == "Bearer sk-from-env" + assert exchange.call_count == 0 + + def test_process_wide_key_still_serves_deployments_without_identity( + self, + deployment_wif: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + surface: str, + headers_from: HeadersFromParams, + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + headers: Final = headers_from(GenericLiteLLMParams()) + assert headers["Authorization"] == "Bearer sk-from-env" + + def test_partial_deployment_identity_fails_loudly( + self, deployment_wif: dict[str, str], surface: str, headers_from: HeadersFromParams + ) -> None: + subset: Final = {key: value for key, value in deployment_wif.items() if key != "openai_service_account_id"} + with pytest.raises(OpenAIError, match="missing openai_service_account_id"): + headers_from(GenericLiteLLMParams(**subset)) + class TestDiscoverModels: @staticmethod @@ -650,6 +990,967 @@ class TestClientsideBaseOverride: ) +IMAGES_URL: Final = "https://api.openai.com/v1/images/generations" +IMAGE_VARIATIONS_URL: Final = "https://api.openai.com/v1/images/variations" +SPEECH_URL: Final = "https://api.openai.com/v1/audio/speech" +TRANSCRIPTIONS_URL: Final = "https://api.openai.com/v1/audio/transcriptions" +MODERATIONS_URL: Final = "https://api.openai.com/v1/moderations" +COMPLETIONS_URL: Final = "https://api.openai.com/v1/completions" +FILES_URL: Final = "https://api.openai.com/v1/files" +BATCHES_URL: Final = "https://api.openai.com/v1/batches" +FINE_TUNING_JOBS_URL: Final = "https://api.openai.com/v1/fine_tuning/jobs" +ASSISTANTS_URL: Final = "https://api.openai.com/v1/assistants" +THREADS_URL: Final = "https://api.openai.com/v1/threads" +CONTAINERS_URL: Final = "https://api.openai.com/v1/containers" +IMAGE_BODY: Final = {"created": 1, "data": [{"b64_json": "aGk="}]} +MODERATION_BODY: Final = { + "id": "modr-1", + "model": "omni-moderation-latest", + "results": [{"flagged": False, "categories": {}, "category_scores": {}}], +} +TEXT_COMPLETION_BODY: Final = { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "ok", "index": 0, "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +FILE_BODY: Final = { + "id": "file-1", + "object": "file", + "bytes": 2, + "created_at": 1, + "filename": "in.jsonl", + "purpose": "batch", + "status": "processed", +} +FILE_DELETED_BODY: Final = {"id": "file-1", "object": "file", "deleted": True} +BATCH_BODY: Final = { + "id": "batch-1", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-1", + "completion_window": "24h", + "status": "validating", + "created_at": 1, +} +FINE_TUNING_JOB_BODY: Final = { + "id": "ftjob-1", + "object": "fine_tuning.job", + "created_at": 1, + "error": None, + "fine_tuned_model": None, + "finished_at": None, + "hyperparameters": {"n_epochs": "auto"}, + "model": "gpt-5.4-nano", + "organization_id": "org-1", + "result_files": [], + "seed": 0, + "status": "queued", + "trained_tokens": None, + "training_file": "file-1", + "validation_file": None, +} + + +ASSISTANT_BODY: Final = { + "id": "asst-1", + "object": "assistant", + "created_at": 1, + "name": "wif", + "description": None, + "model": "gpt-5.4-nano", + "instructions": None, + "tools": [], + "metadata": {}, +} +ASSISTANT_DELETED_BODY: Final = {"id": "asst-1", "object": "assistant.deleted", "deleted": True} +THREAD_BODY: Final = {"id": "thread-1", "object": "thread", "created_at": 1, "metadata": {}} +MESSAGE_BODY: Final = { + "id": "msg-1", + "object": "thread.message", + "created_at": 1, + "thread_id": "thread-1", + "role": "user", + "status": "completed", + "content": [{"type": "text", "text": {"value": "hi", "annotations": []}}], + "assistant_id": None, + "run_id": None, + "attachments": [], + "metadata": {}, +} +RUN_BODY: Final = { + "id": "run-1", + "object": "thread.run", + "created_at": 1, + "thread_id": "thread-1", + "assistant_id": "asst-1", + "status": "completed", + "model": "gpt-5.4-nano", + "instructions": "", + "tools": [], + "metadata": {}, + "parallel_tool_calls": False, +} + + +def mock_streaming_text_completions() -> respx.Route: + events: Final = ( + { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "ok", "index": 0, "finish_reason": None, "logprobs": None}], + }, + { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "", "index": 0, "finish_reason": "stop", "logprobs": None}], + }, + ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + return respx.post(COMPLETIONS_URL).mock( + return_value=httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + + +def bearer_of(route: respx.Route) -> str: + return route.calls.last.request.headers["Authorization"] + + +class TestDeploymentNonChatSurfaces: + @respx.mock + def test_image_generation_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("image-bearer") + route: Final = respx.post(IMAGES_URL).mock(return_value=httpx.Response(200, json=IMAGE_BODY)) + + response: Final = litellm.image_generation(model="openai/gpt-image-2", prompt="a cat", **deployment_wif) + + assert response.data[0].b64_json == "aGk=" + assert bearer_of(route) == "Bearer image-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_image_generation_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("aimage-bearer") + route: Final = respx.post(IMAGES_URL).mock(return_value=httpx.Response(200, json=IMAGE_BODY)) + + response: Final = await litellm.aimage_generation(model="openai/gpt-image-2", prompt="a cat", **deployment_wif) + + assert response.data[0].b64_json == "aGk=" + assert bearer_of(route) == "Bearer aimage-bearer" + + @respx.mock + def test_image_variation_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("variation-bearer") + route: Final = respx.post(IMAGE_VARIATIONS_URL).mock(return_value=httpx.Response(200, json=IMAGE_BODY)) + + response: Final = litellm.image_variation( + model="openai/dall-e-2", image=b"png-bytes", custom_llm_provider="openai", **deployment_wif + ) + + assert response.data[0].b64_json == "aGk=" + assert bearer_of(route) == "Bearer variation-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_image_variation_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("avariation-bearer") + route: Final = respx.post(IMAGE_VARIATIONS_URL).mock(return_value=httpx.Response(200, json=IMAGE_BODY)) + + response: Final = await litellm.aimage_variation( + model="openai/dall-e-2", image=b"png-bytes", custom_llm_provider="openai", **deployment_wif + ) + + assert response.data[0].b64_json == "aGk=" + assert bearer_of(route) == "Bearer avariation-bearer" + + @respx.mock + def test_speech_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("speech-bearer") + route: Final = respx.post(SPEECH_URL).mock( + return_value=httpx.Response(200, content=b"audio", headers={"content-type": "audio/mpeg"}) + ) + + response: Final = litellm.speech(model="openai/gpt-4o-mini-tts", input="hi", voice="alloy", **deployment_wif) + + assert response.content == b"audio" + assert bearer_of(route) == "Bearer speech-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_speech_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("aspeech-bearer") + route: Final = respx.post(SPEECH_URL).mock( + return_value=httpx.Response(200, content=b"audio", headers={"content-type": "audio/mpeg"}) + ) + + response: Final = await litellm.aspeech( + model="openai/gpt-4o-mini-tts", input="hi", voice="alloy", **deployment_wif + ) + + assert response.content == b"audio" + assert bearer_of(route) == "Bearer aspeech-bearer" + + @respx.mock + def test_transcription_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str], tmp_path: Path) -> None: + mock_token_exchange("transcribe-bearer") + route: Final = respx.post(TRANSCRIPTIONS_URL).mock(return_value=httpx.Response(200, json={"text": "hi"})) + audio: Final = tmp_path / "tone.wav" + audio.write_bytes(b"RIFF....WAVE") + + with audio.open("rb") as audio_file: + response: Final = litellm.transcription( + model="openai/gpt-4o-mini-transcribe", file=audio_file, **deployment_wif + ) + + assert response.text == "hi" + assert bearer_of(route) == "Bearer transcribe-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_transcription_kwargs_carry_exchanged_bearer( + self, deployment_wif: dict[str, str], tmp_path: Path + ) -> None: + mock_token_exchange("atranscribe-bearer") + route: Final = respx.post(TRANSCRIPTIONS_URL).mock(return_value=httpx.Response(200, json={"text": "hi"})) + audio: Final = tmp_path / "tone.wav" + audio.write_bytes(b"RIFF....WAVE") + + with audio.open("rb") as audio_file: + response: Final = await litellm.atranscription( + model="openai/gpt-4o-mini-transcribe", file=audio_file, **deployment_wif + ) + + assert response.text == "hi" + assert bearer_of(route) == "Bearer atranscribe-bearer" + + @respx.mock + def test_moderation_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("moderation-bearer") + route: Final = respx.post(MODERATIONS_URL).mock(return_value=httpx.Response(200, json=MODERATION_BODY)) + + response: Final = litellm.moderation(input="hi", model="omni-moderation-latest", **deployment_wif) + + assert response.results[0].flagged is False + assert bearer_of(route) == "Bearer moderation-bearer" + + @respx.mock + def test_moderation_ignores_unrelated_kwargs_it_never_validated(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("moderation-bearer") + route: Final = respx.post(MODERATIONS_URL).mock(return_value=httpx.Response(200, json=MODERATION_BODY)) + + response: Final = litellm.moderation( + input="hi", model="omni-moderation-latest", api_version=20240101, **deployment_wif + ) + + assert response.results[0].flagged is False + assert bearer_of(route) == "Bearer moderation-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_moderation_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("amoderation-bearer") + route: Final = respx.post(MODERATIONS_URL).mock(return_value=httpx.Response(200, json=MODERATION_BODY)) + + response: Final = await litellm.amoderation(input="hi", model="omni-moderation-latest", **deployment_wif) + + assert response.results[0].flagged is False + assert bearer_of(route) == "Bearer amoderation-bearer" + + @respx.mock + def test_text_completion_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("text-bearer") + route: Final = respx.post(COMPLETIONS_URL).mock(return_value=httpx.Response(200, json=TEXT_COMPLETION_BODY)) + + response: Final = litellm.text_completion( + model="text-completion-openai/gpt-3.5-turbo-instruct", prompt="hi", **deployment_wif + ) + + assert response.choices[0].text == "ok" + assert bearer_of(route) == "Bearer text-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_text_completion_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("atext-bearer") + route: Final = respx.post(COMPLETIONS_URL).mock(return_value=httpx.Response(200, json=TEXT_COMPLETION_BODY)) + + response: Final = await litellm.atext_completion( + model="text-completion-openai/gpt-3.5-turbo-instruct", prompt="hi", **deployment_wif + ) + + assert response.choices[0].text == "ok" + assert bearer_of(route) == "Bearer atext-bearer" + + @respx.mock + def test_streaming_text_completion_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("text-stream-bearer") + route: Final = mock_streaming_text_completions() + + chunks: Final = tuple( + litellm.text_completion( + model="text-completion-openai/gpt-3.5-turbo-instruct", prompt="hi", stream=True, **deployment_wif + ) + ) + + assert chunks[0].choices[0].text == "ok" + assert bearer_of(route) == "Bearer text-stream-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_streaming_text_completion_kwargs_carry_exchanged_bearer( + self, deployment_wif: dict[str, str] + ) -> None: + mock_token_exchange("atext-stream-bearer") + route: Final = mock_streaming_text_completions() + + stream: Final = await litellm.atext_completion( + model="text-completion-openai/gpt-3.5-turbo-instruct", prompt="hi", stream=True, **deployment_wif + ) + chunks: Final = [chunk async for chunk in stream] + + assert chunks[0].choices[0].text == "ok" + assert bearer_of(route) == "Bearer atext-stream-bearer" + + @respx.mock + @pytest.mark.parametrize( + ("method", "url", "response", "invoke"), + [ + pytest.param( + "POST", + FILES_URL, + httpx.Response(200, json=FILE_BODY), + lambda p: litellm.create_file(file=("in.jsonl", b"{}"), purpose="batch", **p), + id="create_file", + ), + pytest.param( + "GET", + f"{FILES_URL}/file-1", + httpx.Response(200, json=FILE_BODY), + lambda p: litellm.file_retrieve(file_id="file-1", **p), + id="file_retrieve", + ), + pytest.param( + "DELETE", + f"{FILES_URL}/file-1", + httpx.Response(200, json=FILE_DELETED_BODY), + lambda p: litellm.file_delete(file_id="file-1", **p), + id="file_delete", + ), + pytest.param( + "GET", + FILES_URL, + httpx.Response(200, json={"object": "list", "data": [FILE_BODY]}), + lambda p: litellm.file_list(**p), + id="file_list", + ), + pytest.param( + "GET", + f"{FILES_URL}/file-1/content", + httpx.Response(200, content=b"{}"), + lambda p: litellm.file_content(file_id="file-1", **p), + id="file_content", + ), + pytest.param( + "POST", + BATCHES_URL, + httpx.Response(200, json=BATCH_BODY), + lambda p: litellm.create_batch( + completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-1", **p + ), + id="create_batch", + ), + pytest.param( + "GET", + f"{BATCHES_URL}/batch-1", + httpx.Response(200, json=BATCH_BODY), + lambda p: litellm.retrieve_batch(batch_id="batch-1", **p), + id="retrieve_batch", + ), + pytest.param( + "POST", + f"{BATCHES_URL}/batch-1/cancel", + httpx.Response(200, json=BATCH_BODY), + lambda p: litellm.cancel_batch(batch_id="batch-1", **p), + id="cancel_batch", + ), + pytest.param( + "GET", + BATCHES_URL, + httpx.Response(200, json={"object": "list", "data": [BATCH_BODY], "has_more": False}), + lambda p: litellm.list_batches(**p), + id="list_batches", + ), + pytest.param( + "POST", + FINE_TUNING_JOBS_URL, + httpx.Response(200, json=FINE_TUNING_JOB_BODY), + lambda p: litellm.create_fine_tuning_job(model="gpt-5.4-nano", training_file="file-1", **p), + id="create_fine_tuning_job", + ), + pytest.param( + "POST", + f"{FINE_TUNING_JOBS_URL}/ftjob-1/cancel", + httpx.Response(200, json=FINE_TUNING_JOB_BODY), + lambda p: litellm.cancel_fine_tuning_job(fine_tuning_job_id="ftjob-1", **p), + id="cancel_fine_tuning_job", + ), + pytest.param( + "GET", + FINE_TUNING_JOBS_URL, + httpx.Response(200, json={"object": "list", "data": [FINE_TUNING_JOB_BODY], "has_more": False}), + lambda p: litellm.list_fine_tuning_jobs(**p), + id="list_fine_tuning_jobs", + ), + pytest.param( + "GET", + f"{FINE_TUNING_JOBS_URL}/ftjob-1", + httpx.Response(200, json=FINE_TUNING_JOB_BODY), + lambda p: litellm.retrieve_fine_tuning_job(fine_tuning_job_id="ftjob-1", **p), + id="retrieve_fine_tuning_job", + ), + pytest.param( + "GET", + ASSISTANTS_URL, + httpx.Response(200, json={"object": "list", "data": [ASSISTANT_BODY]}), + lambda p: litellm.get_assistants(**p), + id="get_assistants", + ), + pytest.param( + "POST", + ASSISTANTS_URL, + httpx.Response(200, json=ASSISTANT_BODY), + lambda p: litellm.create_assistants(model="gpt-5.4-nano", **p), + id="create_assistants", + ), + pytest.param( + "DELETE", + f"{ASSISTANTS_URL}/asst-1", + httpx.Response(200, json=ASSISTANT_DELETED_BODY), + lambda p: litellm.delete_assistant(assistant_id="asst-1", **p), + id="delete_assistant", + ), + pytest.param( + "POST", + THREADS_URL, + httpx.Response(200, json=THREAD_BODY), + lambda p: litellm.create_thread(**p), + id="create_thread", + ), + pytest.param( + "GET", + f"{THREADS_URL}/thread-1", + httpx.Response(200, json=THREAD_BODY), + lambda p: litellm.get_thread(thread_id="thread-1", **p), + id="get_thread", + ), + pytest.param( + "POST", + f"{THREADS_URL}/thread-1/messages", + httpx.Response(200, json=MESSAGE_BODY), + lambda p: litellm.add_message(thread_id="thread-1", role="user", content="hi", **p), + id="add_message", + ), + pytest.param( + "GET", + f"{THREADS_URL}/thread-1/messages", + httpx.Response(200, json={"object": "list", "data": [MESSAGE_BODY]}), + lambda p: litellm.get_messages(thread_id="thread-1", **p), + id="get_messages", + ), + ], + ) + def test_managed_object_kwargs_carry_exchanged_bearer( + self, + deployment_wif: dict[str, str], + method: str, + url: str, + response: httpx.Response, + invoke: Callable[[dict[str, str]], object], + ) -> None: + mock_token_exchange("managed-bearer") + route: Final = respx.route(method=method, url__startswith=url).mock(return_value=response) + + invoke({"custom_llm_provider": "openai", **deployment_wif}) + + assert bearer_of(route) == "Bearer managed-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_managed_object_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("amanaged-bearer") + files_route: Final = respx.get(url__startswith=FILES_URL).mock( + return_value=httpx.Response(200, json={"object": "list", "data": [FILE_BODY]}) + ) + batches_route: Final = respx.get(url__startswith=BATCHES_URL).mock( + return_value=httpx.Response(200, json={"object": "list", "data": [BATCH_BODY], "has_more": False}) + ) + jobs_route: Final = respx.get(url__startswith=FINE_TUNING_JOBS_URL).mock( + return_value=httpx.Response(200, json={"object": "list", "data": [FINE_TUNING_JOB_BODY], "has_more": False}) + ) + + await litellm.afile_list(custom_llm_provider="openai", **deployment_wif) + await litellm.alist_batches(custom_llm_provider="openai", **deployment_wif) + await litellm.alist_fine_tuning_jobs(custom_llm_provider="openai", **deployment_wif) + + assert bearer_of(files_route) == "Bearer amanaged-bearer" + assert bearer_of(batches_route) == "Bearer amanaged-bearer" + assert bearer_of(jobs_route) == "Bearer amanaged-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_repeated_managed_object_calls_exchange_the_token_once(self, deployment_wif: dict[str, str]) -> None: + exchange_route: Final = mock_token_exchange("once-bearer") + files_route: Final = respx.get(url__startswith=FILES_URL).mock( + return_value=httpx.Response(200, json={"object": "list", "data": [FILE_BODY]}) + ) + + for _ in range(3): + await litellm.afile_list(custom_llm_provider="openai", **deployment_wif) + + assert files_route.call_count == 3 + assert exchange_route.call_count == 1 + assert bearer_of(files_route) == "Bearer once-bearer" + + @respx.mock + def test_run_thread_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("run-bearer") + create_route: Final = respx.post(f"{THREADS_URL}/thread-1/runs").mock( + return_value=httpx.Response(200, json=RUN_BODY) + ) + poll_route: Final = respx.get(url__startswith=f"{THREADS_URL}/thread-1/runs/run-1").mock( + return_value=httpx.Response(200, json=RUN_BODY) + ) + + response: Final = litellm.run_thread( + custom_llm_provider="openai", thread_id="thread-1", assistant_id="asst-1", **deployment_wif + ) + + assert response.status == "completed" + assert bearer_of(create_route) == "Bearer run-bearer" + assert bearer_of(poll_route) == "Bearer run-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_async_assistant_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("aassistant-bearer") + assistants_route: Final = respx.get(url__startswith=ASSISTANTS_URL).mock( + return_value=httpx.Response(200, json={"object": "list", "data": [ASSISTANT_BODY]}) + ) + threads_route: Final = respx.post(url__startswith=THREADS_URL).mock( + return_value=httpx.Response(200, json=THREAD_BODY) + ) + + await litellm.aget_assistants(custom_llm_provider="openai", **deployment_wif) + await litellm.acreate_thread(custom_llm_provider="openai", **deployment_wif) + + assert bearer_of(assistants_route) == "Bearer aassistant-bearer" + assert bearer_of(threads_route) == "Bearer aassistant-bearer" + + @respx.mock + def test_container_file_kwargs_carry_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("container-files-bearer") + route: Final = respx.get(f"{CONTAINERS_URL}/cntr_1/files").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [], "has_more": False}) + ) + + response: Final = list_container_files(container_id="cntr_1", custom_llm_provider="openai", **deployment_wif) + + assert response.has_more is False + assert bearer_of(route) == "Bearer container-files-bearer" + + @respx.mock + def test_repeated_moderation_calls_exchange_the_token_once(self, deployment_wif: dict[str, str]) -> None: + exchange_route: Final = mock_token_exchange("once-bearer") + moderation_route: Final = respx.post(MODERATIONS_URL).mock( + return_value=httpx.Response(200, json=MODERATION_BODY) + ) + + for _ in range(3): + litellm.moderation(input="hi", model="omni-moderation-latest", **deployment_wif) + + assert moderation_route.call_count == 3 + assert exchange_route.call_count == 1 + + +EMBEDDING_BODY: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, +} +DECISIONS_URL: Final = "https://api.openai.com/v1/decisions" +DECISIONS_BODY: Final = { + "model": "gpt-6-luna", + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}], + "usage": { + "input_tokens": 3, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 4, + }, +} +INPUT_TOKENS_URL: Final = "https://api.openai.com/v1/responses/input_tokens" +DEPLOYMENT_KEY_ROUTES: Final = ( + pytest.param( + CHAT_COMPLETIONS_URL, + httpx.Response(200, json=CHAT_COMPLETION_BODY), + lambda p: litellm.completion(model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "hi"}], **p), + id="completion", + ), + pytest.param( + EMBEDDINGS_URL, + httpx.Response(200, json=EMBEDDING_BODY), + lambda p: litellm.embedding(model="openai/text-embedding-3-small", input=["hi"], **p), + id="embedding", + ), + pytest.param( + IMAGES_URL, + httpx.Response(200, json=IMAGE_BODY), + lambda p: litellm.image_generation(model="openai/gpt-image-2", prompt="a cat", **p), + id="image_generation", + ), + pytest.param( + TRANSCRIPTIONS_URL, + httpx.Response(200, json={"text": "hi"}), + lambda p: litellm.transcription(model="openai/gpt-4o-mini-transcribe", file=("tone.wav", b"RIFF"), **p), + id="transcription", + ), + pytest.param( + MODERATIONS_URL, + httpx.Response(200, json=MODERATION_BODY), + lambda p: litellm.moderation(input="hi", model="omni-moderation-latest", **p), + id="moderation", + ), + pytest.param( + MODERATIONS_URL, + httpx.Response(200, json=MODERATION_BODY), + lambda p: asyncio.run(litellm.amoderation(input="hi", model="omni-moderation-latest", **p)), + id="amoderation", + ), + pytest.param( + ASSISTANTS_URL, + httpx.Response(200, json={"object": "list", "data": [ASSISTANT_BODY]}), + lambda p: litellm.get_assistants(custom_llm_provider="openai", **p), + id="get_assistants", + ), + pytest.param( + ASSISTANTS_URL, + httpx.Response(200, json=ASSISTANT_BODY), + lambda p: litellm.create_assistants(custom_llm_provider="openai", model="gpt-5.4-nano", **p), + id="create_assistants", + ), + pytest.param( + f"{ASSISTANTS_URL}/asst-1", + httpx.Response(200, json=ASSISTANT_DELETED_BODY), + lambda p: litellm.delete_assistant(custom_llm_provider="openai", assistant_id="asst-1", **p), + id="delete_assistant", + ), + pytest.param( + DECISIONS_URL, + httpx.Response(200, json=DECISIONS_BODY), + lambda p: litellm.decisions( + model="openai/gpt-6-luna", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + **p, + ), + id="decisions", + ), +) + + +@pytest.mark.parametrize(("url", "response", "invoke"), DEPLOYMENT_KEY_ROUTES) +class TestDeploymentKeyBesideIdentity: + @respx.mock + def test_deployment_key_equal_to_the_process_key_wins_without_an_exchange( + self, + deployment_wif: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + url: str, + response: httpx.Response, + invoke: Callable[[dict[str, str]], object], + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-shared") + exchange: Final = mock_token_exchange() + route: Final = respx.route(url__startswith=url).mock(return_value=response) + + invoke({"api_key": "sk-shared", **deployment_wif}) + + assert bearer_of(route) == "Bearer sk-shared" + assert exchange.call_count == 0 + + @respx.mock + def test_process_key_beats_deployment_identity_without_an_exchange( + self, + deployment_wif: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + url: str, + response: httpx.Response, + invoke: Callable[[dict[str, str]], object], + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-shared") + exchange: Final = mock_token_exchange() + route: Final = respx.route(url__startswith=url).mock(return_value=response) + + invoke(deployment_wif) + + assert bearer_of(route) == "Bearer sk-shared" + assert exchange.call_count == 0 + + @respx.mock + def test_secret_manager_key_beats_deployment_identity_without_an_exchange( + self, + deployment_wif: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + url: str, + response: httpx.Response, + invoke: Callable[[dict[str, str]], object], + ) -> None: + monkeypatch.setattr(litellm, "secret_manager_client", _VaultHoldingOpenAIKey()) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + exchange: Final = mock_token_exchange() + route: Final = respx.route(url__startswith=url).mock(return_value=response) + + invoke(deployment_wif) + + assert bearer_of(route) == "Bearer sk-from-vault" + assert exchange.call_count == 0 + + +class _VaultHoldingOpenAIKey(CustomSecretManager): + def __init__(self) -> None: + super().__init__(secret_manager_name="openai_key_vault") + + async def async_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return self.sync_read_secret(secret_name, optional_params, timeout) + + def sync_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return "sk-from-vault" if secret_name == "OPENAI_API_KEY" else None + + +class TestCountTokensFromDeployment: + @staticmethod + async def count(litellm_params: dict[str, str]) -> int | None: + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={"litellm_params": litellm_params}, + ) + return None if result is None else result.total_tokens + + @respx.mock + @pytest.mark.asyncio + async def test_keyless_deployment_counts_with_the_exchanged_bearer(self, deployment_wif: dict[str, str]) -> None: + mock_token_exchange("count-bearer") + route: Final = respx.post(INPUT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 7})) + + assert await self.count(deployment_wif) == 7 + assert bearer_of(route) == "Bearer count-bearer" + + @respx.mock + @pytest.mark.asyncio + async def test_process_key_beats_deployment_identity_without_an_exchange( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + exchange: Final = mock_token_exchange() + route: Final = respx.post(INPUT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 7})) + + await self.count(deployment_wif) + + assert bearer_of(route) == "Bearer sk-from-env" + assert exchange.call_count == 0 + + @respx.mock + @pytest.mark.asyncio + @pytest.mark.parametrize("module_key", ["api_key", "openai_key"]) + async def test_module_level_key_beats_deployment_identity_without_an_exchange( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch, module_key: str + ) -> None: + monkeypatch.setattr(litellm, module_key, "sk-module-level") + exchange: Final = mock_token_exchange() + route: Final = respx.post(INPUT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 7})) + + await self.count(deployment_wif) + + assert bearer_of(route) == "Bearer sk-module-level" + assert exchange.call_count == 0 + + @respx.mock + @pytest.mark.asyncio + async def test_deployment_key_wins_without_an_exchange(self, deployment_wif: dict[str, str]) -> None: + exchange: Final = mock_token_exchange() + route: Final = respx.post(INPUT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 7})) + + await self.count({"api_key": "sk-count", **deployment_wif}) + + assert bearer_of(route) == "Bearer sk-count" + assert exchange.call_count == 0 + + @respx.mock + @pytest.mark.asyncio + @pytest.mark.parametrize( + "identity_keys", + [tuple(OPENAI_WIF_KWARGS_KEYS), ("openai_identity_provider_id",)], + ids=["complete_trio", "partial_trio"], + ) + async def test_secret_manager_key_beats_deployment_identity_without_an_exchange( + self, deployment_wif: dict[str, str], monkeypatch: pytest.MonkeyPatch, identity_keys: tuple[str, ...] + ) -> None: + monkeypatch.setattr(litellm, "secret_manager_client", _VaultHoldingOpenAIKey()) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + exchange: Final = mock_token_exchange() + route: Final = respx.post(INPUT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 7})) + + assert await self.count({key: deployment_wif[key] for key in identity_keys}) == 7 + assert bearer_of(route) == "Bearer sk-from-vault" + assert exchange.call_count == 0 + + @respx.mock + @pytest.mark.asyncio + async def test_partial_deployment_identity_answers_a_401_naming_the_gap_without_a_count_call(self) -> None: + exchange: Final = mock_token_exchange() + route: Final = respx.post(INPUT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 7})) + + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={"litellm_params": {"openai_identity_provider_id": "idp-only"}}, + ) + + assert result is not None + assert (result.error, result.status_code, result.total_tokens) == (True, 401, 0) + assert "openai_service_account_id" in (result.error_message or "") + assert route.call_count == 0 + assert exchange.call_count == 0 + + +VECTOR_STORES_URL: Final = "https://api.openai.com/v1/vector_stores" +LIST_PAGE_BODY: Final = {"object": "list", "data": [], "has_more": False} + + +def mock_token_exchange_naming_its_thread(event_loop_thread: int) -> respx.Route: + def respond(request: httpx.Request) -> httpx.Response: + minted_on: Final = "event-loop" if threading.get_ident() == event_loop_thread else "worker-thread" + return httpx.Response(200, json={"access_token": f"minted-on-{minted_on}", "expires_in": 3600}) + + return respx.post(TOKEN_EXCHANGE_URL).mock(side_effect=respond) + + +LOOP_BOUND_ROUTES: Final = ( + pytest.param( + "POST", + "https://api.openai.com/v1/images/edits", + httpx.Response(200, json=IMAGE_BODY), + lambda p: litellm.aimage_edit( + model="openai/gpt-image-2", image=[io.BytesIO(b"png-bytes")], prompt="add a hat", **p + ), + id="image_edit", + ), + pytest.param( + "GET", + "https://api.openai.com/v1/videos/video_1", + httpx.Response( + 200, json={"id": "video_1", "object": "video", "status": "completed", "created_at": 1, "model": "sora-2"} + ), + lambda p: litellm.avideo_status(video_id="video_1", custom_llm_provider="openai", **p), + id="video_status", + ), + pytest.param( + "GET", + f"{CONTAINERS_URL}/cntr_1", + httpx.Response( + 200, + json={ + "id": "cntr_1", + "object": "container", + "created_at": 1, + "status": "running", + "expires_after": {"anchor": "last_active_at", "minutes": 20}, + "last_active_at": 1, + "name": "sandbox", + }, + ), + lambda p: aretrieve_container(container_id="cntr_1", custom_llm_provider="openai", **p), + id="retrieve_container", + ), + pytest.param( + "GET", + f"{CONTAINERS_URL}/cntr_1/files", + httpx.Response(200, json=LIST_PAGE_BODY), + lambda p: alist_container_files(container_id="cntr_1", custom_llm_provider="openai", **p), + id="container_files", + ), + pytest.param( + "POST", + f"{VECTOR_STORES_URL}/vs_1/search", + httpx.Response(200, json={"object": "vector_store.search_results.page", "search_query": "q", "data": []}), + lambda p: litellm.vector_stores.asearch(vector_store_id="vs_1", query="q", custom_llm_provider="openai", **p), + id="vector_store_search", + ), + pytest.param( + "GET", + f"{VECTOR_STORES_URL}/vs_1/files", + httpx.Response(200, json=LIST_PAGE_BODY), + lambda p: litellm.vector_store_files.alist(vector_store_id="vs_1", custom_llm_provider="openai", **p), + id="vector_store_files", + ), + pytest.param( + "POST", + DECISIONS_URL, + httpx.Response(200, json=DECISIONS_BODY), + lambda p: litellm.adecisions( + model="openai/gpt-6-luna", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + **p, + ), + id="decisions", + ), +) + + +@pytest.mark.parametrize(("method", "url", "response", "invoke"), LOOP_BOUND_ROUTES) +class TestTokenExchangeLeavesTheEventLoopFree: + @respx.mock + @pytest.mark.asyncio + async def test_token_exchange_runs_off_the_event_loop( + self, + deployment_wif: dict[str, str], + method: str, + url: str, + response: httpx.Response, + invoke: Callable[[dict[str, str]], Awaitable[object]], + ) -> None: + mock_token_exchange_naming_its_thread(threading.get_ident()) + upstream: Final = respx.route(method=method, url=url).mock(return_value=response) + + await invoke(deployment_wif) + + assert bearer_of(upstream) == "Bearer minted-on-worker-thread" + + class TestCountTokens: @pytest.mark.asyncio @respx.mock diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index c224cd08663..7de249bfb6e 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -35,7 +35,7 @@ from litellm.proxy.auth.auth_utils import ( is_request_body_safe, ) from litellm.router import Router -from litellm.types.utils import oauth_token_exchange_litellm_params +from litellm.types.utils import oauth_token_exchange_litellm_params, server_owned_wif_litellm_params from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS @@ -163,13 +163,45 @@ def test_every_server_owned_identity_param_is_refused_from_a_request_body(param: ) +def _client_router_config(placement: str, litellm_params: dict[str, str]) -> dict[str, object]: + if placement == "model_list": + return {"model_list": [{"model_name": "gpt-5.4-nano", "litellm_params": litellm_params}]} + return {"default_litellm_params": litellm_params} + + +@pytest.mark.parametrize("param", server_owned_wif_litellm_params) +@pytest.mark.parametrize("placement", ["model_list", "default_litellm_params"]) +def test_every_server_owned_auth_param_is_refused_from_a_client_router_config(param: str, placement: str): + user_config: Final = _client_router_config(placement, {"model": "openai/gpt-5.4-nano", param: "attacker-chosen"}) + with pytest.raises( + ValueError, + match="server-owned workload identity federation or OAuth token exchange parameter", + ): + is_request_body_safe( + request_body={"model": "gpt-5.4-nano", "user_config": user_config}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-5.4-nano", + ) + + @pytest.mark.parametrize( "body", [ {"model": "claude-sonnet-5", "litellm_credential_name": "admin-wif"}, {"model": "claude-sonnet-5", "litellm_params": {"litellm_credential_name": "admin-wif"}}, + { + "model": "claude-sonnet-5", + "user_config": _client_router_config( + "model_list", {"model": "anthropic/claude-sonnet-5", "litellm_credential_name": "admin-wif"} + ), + }, + { + "model": "claude-sonnet-5", + "user_config": _client_router_config("default_litellm_params", {"litellm_credential_name": "admin-wif"}), + }, ], - ids=["top_level", "nested_litellm_params"], + ids=["top_level", "nested_litellm_params", "client_router_model_list", "client_router_defaults"], ) def test_a_request_body_cannot_pick_a_federated_identity_by_credential_name(monkeypatch, body: dict): """Naming a federated credential moves the token exchange onto that credential's federation rule diff --git a/tests/unit/proxy/batches_endpoints/test_endpoints.py b/tests/unit/proxy/batches_endpoints/test_endpoints.py index aab2074e49f..15a254dd7a1 100644 --- a/tests/unit/proxy/batches_endpoints/test_endpoints.py +++ b/tests/unit/proxy/batches_endpoints/test_endpoints.py @@ -35,6 +35,7 @@ import logging from contextlib import ExitStack from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Any, Dict, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -521,6 +522,29 @@ async def test_create__fallback_explicit_provider_bypasses_not_found_gate(harnes assert harness.acreate_kwargs()["custom_llm_provider"] == "anthropic" +@pytest.mark.asyncio +async def test_create__fallback_workload_identity_alone_forwards( + harness: Harness, no_openai_creds: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + token_file = tmp_path / "subject_token.jwt" + token_file.write_text("subject-token-from-file") + monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123") + monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456") + monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file)) + set_body( + harness, + { + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + ) + + await call_create(harness) + + assert harness.acreate_kwargs()["custom_llm_provider"] == "openai" + + @pytest.mark.asyncio async def test_create__fallback_env_key_alone_forwards(harness, monkeypatch): monkeypatch.setattr(litellm, "api_key", None) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/unit/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index 2c91569de69..930309d62ed 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -5,6 +5,7 @@ Test OpenAI Moderation Guardrail import json import os +from pathlib import Path from typing import Final @@ -12,13 +13,22 @@ from unittest.mock import MagicMock, patch import httpx import pytest +import respx +import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai.workload_identity import _workload_identity_auth from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import get_logging_caching_headers +from litellm.proxy.guardrails.guardrail_hooks.openai import ( + initialize_guardrail as initialize_openai_moderation_guardrail, +) from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) +from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER, InMemoryGuardrailHandler +from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 +from litellm.types.guardrails import Guardrail, LitellmParams, SupportedGuardrailIntegrations from litellm.types.llms.openai import OpenAIModerationResponse, OpenAIModerationResult @@ -1089,3 +1099,153 @@ async def test_openai_moderation_responses_api_input_field(): input_type="request", ) assert "Violated OpenAI moderation policy" in str(chat_error.value) + + +class _ModerationsEndpointThatRecordsTheAuthorization: + authorization: str | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.authorization = request.headers.get("Authorization") + return httpx.Response( + 200, + json={ + "id": "modr-federated", + "model": "omni-moderation-latest", + "results": [ + {"flagged": False, "categories": {}, "category_scores": {}, "category_applied_input_types": {}} + ], + }, + ) + + +@pytest.mark.asyncio +@respx.mock +async def test_openai_moderation_sends_the_federated_bearer_when_only_workload_identity_is_set( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + token_file: Final = tmp_path / "subject_token.jwt" + token_file.write_text("subject-token") + for name in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE"): + monkeypatch.delenv(name, raising=False) + for attribute in ("api_key", "openai_key", "api_base"): + monkeypatch.setattr(litellm, attribute, None) + monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_guardrail") + monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-guardrail") + monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file)) + _workload_identity_auth.cache_clear() + respx.post("https://auth.openai.com/oauth/token").mock( + return_value=httpx.Response(200, json={"access_token": "guardrail-bearer", "expires_in": 3600}) + ) + endpoint: Final = _ModerationsEndpointThatRecordsTheAuthorization() + http_client: Final = AsyncHTTPHandler() + http_client.client = httpx.AsyncClient(transport=httpx.MockTransport(endpoint)) + + guardrail: Final = OpenAIModerationGuardrail(guardrail_name="openai-mod") + guardrail.async_handler = http_client + response: Final = await guardrail.async_make_request("hello") + + assert response.id == "modr-federated" + assert endpoint.authorization == "Bearer guardrail-bearer" + + +@pytest.mark.asyncio +@respx.mock +async def test_openai_moderation_federates_with_the_identity_in_its_own_guardrail_config( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + token_file: Final = tmp_path / "subject_token.jwt" + token_file.write_text("subject-token") + for name in ( + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_API_BASE", + "OPENAI_IDENTITY_PROVIDER_ID", + "OPENAI_SERVICE_ACCOUNT_ID", + "OPENAI_IDENTITY_TOKEN_FILE", + ): + monkeypatch.delenv(name, raising=False) + for attribute in ("api_key", "openai_key", "api_base"): + monkeypatch.setattr(litellm, attribute, None) + monkeypatch.setenv("GUARDRAIL_IDENTITY_PROVIDER_ID", "idp_guardrail_config") + _workload_identity_auth.cache_clear() + exchange: Final = respx.post("https://auth.openai.com/oauth/token").mock( + return_value=httpx.Response(200, json={"access_token": "guardrail-config-bearer", "expires_in": 3600}) + ) + endpoint: Final = _ModerationsEndpointThatRecordsTheAuthorization() + http_client: Final = AsyncHTTPHandler() + http_client.client = httpx.AsyncClient(transport=httpx.MockTransport(endpoint)) + + guardrail_config: Final = LitellmParams( + guardrail=SupportedGuardrailIntegrations.OPENAI_MODERATION, + mode="pre_call", + openai_identity_provider_id="os.environ/GUARDRAIL_IDENTITY_PROVIDER_ID", + openai_service_account_id="user-guardrail-config", + openai_identity_token_file=str(token_file), + ) + monkeypatch.setattr(litellm, "callbacks", []) + guardrail: Final = initialize_openai_moderation_guardrail( + litellm_params=guardrail_config, + guardrail=Guardrail(guardrail_name="openai-mod", litellm_params=guardrail_config), + ) + guardrail.async_handler = http_client + await guardrail.async_make_request("hello") + + exchange_body: Final = json.loads(exchange.calls.last.request.content) + assert exchange_body["identity_provider_id"] == "idp_guardrail_config" + assert exchange_body["service_account_id"] == "user-guardrail-config" + assert endpoint.authorization == "Bearer guardrail-config-bearer" + + +@pytest.fixture +def no_ambient_openai_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_API_BASE", + "OPENAI_IDENTITY_PROVIDER_ID", + "OPENAI_SERVICE_ACCOUNT_ID", + "OPENAI_IDENTITY_TOKEN_FILE", + ): + monkeypatch.delenv(name, raising=False) + for attribute in ("api_key", "openai_key", "api_base"): + monkeypatch.setattr(litellm, attribute, None) + monkeypatch.setattr(litellm, "callbacks", []) + + +PARTIAL_IDENTITY_MODERATION: Final = { + "guardrail": SupportedGuardrailIntegrations.OPENAI_MODERATION.value, + "mode": "pre_call", + "openai_identity_provider_id": "idp_without_the_rest", +} + + +@pytest.mark.usefixtures("no_ambient_openai_credentials") +def test_openai_moderation_with_a_partial_identity_is_a_configuration_error_naming_the_gap() -> None: + with pytest.raises(ValueError, match="openai_service_account_id, openai_identity_token_file"): + InMemoryGuardrailHandler().initialize_guardrail( + guardrail={"guardrail_name": "partial-identity-mod", "litellm_params": PARTIAL_IDENTITY_MODERATION}, + source="db", + ) + + +@pytest.mark.usefixtures("no_ambient_openai_credentials") +def test_proxy_boot_skips_an_openai_moderation_with_a_partial_identity() -> None: + IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.clear() + IN_MEMORY_GUARDRAIL_HANDLER.guardrail_id_to_custom_guardrail.clear() + + init_guardrails_v2( + all_guardrails=[ + {"guardrail_name": "partial-identity-mod", "litellm_params": PARTIAL_IDENTITY_MODERATION}, + { + "guardrail_name": "keyed-mod", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.OPENAI_MODERATION.value, + "mode": "pre_call", + "api_key": "sk-moderation", + }, + }, + ] + ) + + booted: Final = IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.values() + assert {guardrail["guardrail_name"] for guardrail in booted} == {"keyed-mod"} diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 41179967a7f..527d3d599b1 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -2,17 +2,20 @@ import asyncio import json import time from collections.abc import Awaitable, Callable, Mapping +from pathlib import Path from types import TracebackType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx import litellm from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai.workload_identity import _workload_identity_auth from litellm.models.credentials import CredentialItem from litellm.realtime_api import main as realtime_main from litellm.realtime_api.main import _with_resolved_session_model @@ -1052,3 +1055,124 @@ async def test_voice_transcript_blocked_by_guardrail(): finally: litellm.callbacks = [] + + +@pytest.fixture +def federated_deployment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> dict[str, str]: + token_file: Final = tmp_path / "subject_token.jwt" + token_file.write_text("subject-token") + for name in ( + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_API_BASE", + "OPENAI_IDENTITY_PROVIDER_ID", + "OPENAI_SERVICE_ACCOUNT_ID", + "OPENAI_IDENTITY_TOKEN_FILE", + ): + monkeypatch.delenv(name, raising=False) + for attribute in ("api_key", "openai_key", "api_base"): + monkeypatch.setattr(litellm, attribute, None) + _workload_identity_auth.cache_clear() + return { + "openai_identity_provider_id": "idp_realtime", + "openai_service_account_id": "user-realtime", + "openai_identity_token_file": str(token_file), + } + + +def _mock_token_exchange() -> None: + respx.post("https://auth.openai.com/oauth/token").mock( + return_value=httpx.Response(200, json={"access_token": "federated-bearer", "expires_in": 3600}) + ) + + +class _ConnectThatStopsAfterCapturingTheAuthorization: + authorization: str | None = None + + def __call__( + self, url: str, *, additional_headers: Mapping[str, str], **kwargs: object + ) -> "_ConnectThatStopsAfterCapturingTheAuthorization": + self.authorization = additional_headers.get("Authorization") + return self + + async def __aenter__(self) -> None: + raise RuntimeError("backend headers captured, nothing to bridge") + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + return None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "live_session_start", + [None, {"type": "session.start", "session": {"model": "gpt-live-1"}}], + ids=["realtime", "live_session"], +) +@respx.mock +async def test_openai_websocket_sessions_send_the_federated_bearer_for_a_keyless_deployment( + federated_deployment: dict[str, str], live_session_start: Mapping[str, object] | None +) -> None: + _mock_token_exchange() + connect: Final = _ConnectThatStopsAfterCapturingTheAuthorization() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="gpt-realtime", + custom_llm_provider="openai", + websocket=MagicMock(close=AsyncMock()), + litellm_logging_obj=FakeLogging(), + live_session_start=live_session_start, + **federated_deployment, + ) + assert connect.authorization == "Bearer federated-bearer" + + +class _GatewayThatRecordsTheAuthorization: + authorization: str | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.authorization = request.headers.get("Authorization") + return httpx.Response(200, json={"value": "ek_test"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [realtime_main.acreate_realtime_client_secret, realtime_main.acreate_realtime_transcription_session], + ids=["client_secrets", "transcription_sessions"], +) +@respx.mock +async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyless_deployment( + federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]] +) -> None: + _mock_token_exchange() + gateway: Final = _GatewayThatRecordsTheAuthorization() + await route.__wrapped__( + model="gpt-realtime", + custom_llm_provider="openai", + litellm_logging_obj=FakeLogging(), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + **federated_deployment, + ) + assert gateway.authorization == "Bearer federated-bearer" + + +@pytest.mark.asyncio +@respx.mock +async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_keyless_deployment( + federated_deployment: dict[str, str], +) -> None: + _mock_token_exchange() + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main.realtime_health_check( + model="gpt-realtime", + custom_llm_provider="openai", + api_key=None, + model_params={"model": "openai/gpt-realtime", **federated_deployment}, + ) + assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer federated-bearer"} diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index a5d74092d16..e73b9950594 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -4890,6 +4890,14 @@ LLMS_FORWARDER_CASES: Final = ( "staticmethod", False, ), + ( + "litellm.llms.openai.common_utils", + "BaseOpenAILLM", + "_get_sync_http_client", + "get_sync_http_client", + "staticmethod", + False, + ), ( "litellm.llms.openai.openai", "OpenAIChatCompletion", diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 8092437fbd3..0db122b927f 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -373,3 +373,9 @@ def test_server_owned_wif_fields_present_reports_openai_fields(): def test_reject_server_owned_wif_params_names_each_openai_field(param_name: str): with pytest.raises(ValueError, match=param_name): reject_server_owned_wif_params({param_name: "client-supplied"}) + + +@pytest.mark.parametrize("param_name", openai_wif_litellm_params) +def test_reject_server_owned_wif_params_catches_the_array_form_key(param_name: str): + with pytest.raises(ValueError, match=param_name): + reject_server_owned_wif_params({f"{param_name}[]": "client-supplied"})