scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 174

Merge pull request #50962 from BerriAI/litellm_backport_39966_lts_1_108_x

Browse source 
feat(openai)!: honor per-deployment workload identity on non-chat routes (#39966) [lts/1.108.x]
This commit is contained in:
yuneng-jiang 2026-11-10 10:24:14 -08:00 • committed by GitHub
commit 58c13a5290
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
50 changed files with 2700 additions and 350 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/assistants/main.py
View file

@ -129,6 +129,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_assistants=aget_assistants,
)
@ -312,6 +313,7 @@ def create_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants,
@ -461,6 +463,7 @@ def delete_assistant(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
assistant_id=assistant_id,
client=client,
async_delete_assistants=async_delete_assistants,
@ -637,6 +640,7 @@ def create_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
acreate_thread=acreate_thread,
)
@ -783,6 +787,7 @@ def get_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_thread=aget_thread,
)
@ -960,6 +965,7 @@ def add_message(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
a_add_message=a_add_message,
)
@ -1111,6 +1117,7 @@ def get_messages(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_messages=aget_messages,
)
@ -1308,6 +1315,7 @@ def run_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
arun_thread=arun_thread,
event_handler=event_handler,

3
litellm/batches/batch_utils.py
View file

@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
)
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
from litellm.utils import token_counter
@ -594,6 +594,7 @@ def extract_file_access_credentials(
# A federated deployment holds no api_key, so without these the fetch that reads a
# finished batch's output has nothing to authenticate with and its cost is never billed.
*sorted(ANTHROPIC_WIF_KWARGS_KEYS),
*sorted(OPENAI_WIF_KWARGS_KEYS),
)
for key in credential_keys:
if key in litellm_params:

4
litellm/batches/main.py
View file

@ -309,6 +309,7 @@ def create_batch(
timeout=timeout,
max_retries=optional_params.max_retries,
_is_async=_is_async,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -819,6 +821,7 @@ def list_batches(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -1016,6 +1019,7 @@ def cancel_batch(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

1
litellm/containers/README.md
View file

@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
self,
headers: dict,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""Azure uses api-key header instead of Bearer token."""
import litellm

3
litellm/decisions/call.py
View file

@ -485,7 +485,7 @@ def prepare_call(
) from error
try:
resolved_api_base, resolved_api_key = provider_config.resolve_connection(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or api_base, dynamic_api_key or api_key
)
except ValueError as error:
@ -496,6 +496,7 @@ def prepare_call(
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",

31
litellm/decisions/main.py
View file

@ -1,5 +1,7 @@
import asyncio
import warnings
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
@ -115,19 +117,22 @@ async def adecisions(
videos: DecisionsVideoClips | None = None,
**kwargs: object,
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
call: Final = await asyncio.to_thread(
partial(
_prepare_decisions_call,
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await asend(call)

6
litellm/files/main.py
View file

@ -259,6 +259,7 @@ def create_file(
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -388,6 +389,7 @@ def file_retrieve(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -569,6 +571,7 @@ def file_delete(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -775,6 +778,7 @@ def file_list(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -972,6 +976,7 @@ def file_content(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -1093,6 +1098,7 @@ def file_content_streaming(
organization=openai_creds.organization,
chunk_size=chunk_size,
client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
if not _is_async:

4
litellm/fine_tuning/main.py
View file

@ -239,6 +239,7 @@ def create_fine_tuning_job(
client=kwargs.get(
"client", None
), # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":

9
litellm/images/main.py
View file

@ -38,6 +38,7 @@ from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@ -504,6 +505,7 @@ def image_generation(
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
if model is None:
@ -677,10 +679,13 @@ def image_variation(
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
if api_key is None and (
resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
is None
):
raise ValueError("API key is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,

1
litellm/llms/azure/containers/transformation.py
View file

@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return BaseAzureLLM.base_validate_azure_environment(
headers=headers,

12
litellm/llms/azure/fine_tuning/handler.py
View file

@ -1,4 +1,4 @@
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Final, cast
import httpx
@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
self._ensure_training_type(create_fine_tuning_job_data)
@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
_is_async: bool = False,
api_version: str | None = None,
litellm_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
# Override to use Azure-specific client initialization
if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI):
client = None
return self.get_azure_openai_client(
litellm_params=litellm_params or {},
litellm_params=dict(litellm_params) if litellm_params is not None else None,
api_key=api_key,
api_base=api_base,
api_version=api_version,

1
litellm/llms/base_llm/containers/transformation.py
View file

@ -81,6 +81,7 @@ class BaseContainerConfig(ABC):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return {}

5
litellm/llms/base_llm/decisions/transformation.py
View file

@ -495,6 +495,11 @@ class BaseDecisionsConfig(ABC):
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
return api_key
@staticmethod
def _first_secret(names: tuple[str, ...]) -> str | None:
return next((value for value in (get_secret_str(name) for name in names) if value), None)

4
litellm/llms/custom_httpx/container_handler.py
View file

@ -5,6 +5,7 @@ This module provides a single generic handler that can process any container fil
endpoint defined in endpoints.json, eliminating the need for individual handler methods.
"""
import asyncio
import json
from collections.abc import Coroutine, Mapping, Sequence
from pathlib import Path
@ -236,6 +237,7 @@ def _request_headers(
return container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
@ -435,7 +437,7 @@ class GenericContainerHandler:
http_client: Final = _async_http_client(client, litellm_params)
# Build request
headers = _request_headers(container_provider_config, extra_headers, litellm_params)
headers = await asyncio.to_thread(_request_headers, container_provider_config, extra_headers, litellm_params)
if extra_headers:
headers.update(extra_headers)

135
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -7172,7 +7172,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = image_edit_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
image_edit_provider_config.validate_environment,
api_key=litellm_params.api_key,
headers=image_edit_optional_request_params.get("extra_headers", {}) or {},
model=model,
@ -7631,7 +7632,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_generation_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_generation_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=video_generation_optional_request_params.get("extra_headers", {}) or {},
model=model,
@ -7826,7 +7828,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_content_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_content_provider_config.validate_environment,
headers=extra_headers or {},
model="",
api_key=api_key,
@ -8006,7 +8009,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_remix_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_remix_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8169,7 +8173,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8309,7 +8314,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8481,7 +8487,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8667,7 +8674,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8799,7 +8807,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_list_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_list_provider_config.validate_environment,
api_key=api_key,
headers=extra_headers or {},
model="",
@ -8880,7 +8889,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_delete_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_delete_provider_config.validate_environment,
api_key=api_key,
headers=extra_headers or {},
model="",
@ -9064,7 +9074,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_status_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_status_provider_config.validate_environment,
api_key=api_key,
headers=extra_headers or {},
model="",
@ -9167,6 +9178,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
# Add Content-Type header for JSON requests
@ -9242,9 +9254,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
# Add Content-Type header for JSON requests
@ -9338,6 +9352,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9414,9 +9429,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9505,6 +9522,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9581,9 +9599,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9674,6 +9694,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9750,9 +9771,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9848,6 +9871,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9926,9 +9950,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -10017,6 +10043,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -10092,9 +10119,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -10212,8 +10241,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10455,8 +10486,10 @@ class BaseLLMHTTPHandler:
vector_store_provider_config.validate_create_vector_store()
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10586,8 +10619,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10708,8 +10743,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10850,8 +10887,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10995,8 +11034,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -11113,8 +11154,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -11255,8 +11298,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11396,8 +11441,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11527,8 +11574,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11664,8 +11713,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11807,8 +11858,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)

4
litellm/llms/openai/common_utils.py
View file

@ -414,6 +414,10 @@ class BaseOpenAILLM:
http2=http2_enabled(),
)
@classmethod
def get_sync_http_client(cls) -> httpx.Client | None:
return cls._get_sync_http_client()
class OpenAICredentials(NamedTuple):
api_base: str

71
litellm/llms/openai/completion/handler.py
View file

@ -1,9 +1,7 @@
import json
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Final
from openai import AsyncOpenAI, OpenAI
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -12,7 +10,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
from litellm.utils import ProviderConfigManager
from ..common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
from ..common_utils import BaseOpenAILLM, OpenAIError
from ..workload_identity import build_async_openai_client, build_openai_client
from .transformation import OpenAITextCompletionConfig
@ -43,7 +42,7 @@ class OpenAITextCompletion(BaseLLM):
print_verbose: Callable | None = None,
api_base: str | None = None,
acompletion: bool = False,
litellm_params=None,
litellm_params: Mapping[str, object] | None = None,
logger_fn=None,
client=None,
organization: str | None = None,
@ -95,6 +94,7 @@ class OpenAITextCompletion(BaseLLM):
max_retries=max_retries,
client=client,
organization=organization,
litellm_params=litellm_params,
)
else:
return self.acompletion(
@ -109,6 +109,7 @@ class OpenAITextCompletion(BaseLLM):
max_retries=max_retries,
organization=organization,
client=client,
litellm_params=litellm_params,
)
elif optional_params.get("stream", False):
return self.streaming(
@ -123,19 +124,22 @@ class OpenAITextCompletion(BaseLLM):
max_retries=max_retries,
client=client,
organization=organization,
litellm_params=litellm_params,
)
else:
if client is None:
openai_client = OpenAI(
openai_client: Final = (
client
if client is not None
else build_openai_client(
api_key=api_key,
base_url=api_base,
http_client=litellm.client_session or OpenAIHTTPClient(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.client_session,
)
else:
openai_client = client
)
raw_response: Final = openai_client.completions.with_raw_response.create(**data)
response: Final = raw_response.parse()
@ -175,19 +179,22 @@ class OpenAITextCompletion(BaseLLM):
max_retries: int,
organization: str | None = None,
client=None,
litellm_params: Mapping[str, object] | None = None,
):
try:
if client is None:
openai_aclient = AsyncOpenAI(
openai_aclient: Final = (
client
if client is not None
else build_async_openai_client(
api_key=api_key,
base_url=api_base,
http_client=BaseOpenAILLM.get_async_http_client(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=BaseOpenAILLM.get_async_http_client,
)
else:
openai_aclient = client
)
raw_response: Final = await openai_aclient.completions.with_raw_response.create(**data)
response: Final = raw_response.parse()
@ -228,18 +235,21 @@ class OpenAITextCompletion(BaseLLM):
max_retries=None,
client=None,
organization=None,
litellm_params: Mapping[str, object] | None = None,
):
if client is None:
openai_client = OpenAI(
openai_client: Final = (
client
if client is not None
else build_openai_client(
api_key=api_key,
base_url=api_base,
http_client=litellm.client_session or OpenAIHTTPClient(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.client_session,
)
else:
openai_client = client
)
try:
raw_response: Final = openai_client.completions.with_raw_response.create(**data)
@ -285,18 +295,21 @@ class OpenAITextCompletion(BaseLLM):
api_base: str | None = None,
client=None,
organization=None,
litellm_params: Mapping[str, object] | None = None,
):
if client is None:
openai_client = AsyncOpenAI(
openai_client: Final = (
client
if client is not None
else build_async_openai_client(
api_key=api_key,
base_url=api_base,
http_client=litellm.aclient_session or OpenAIAsyncHTTPClient(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.aclient_session,
)
else:
openai_client = client
)
raw_response: Final = await openai_client.completions.with_raw_response.create(**data)
response: Final = raw_response.parse()

13
litellm/llms/openai/containers/transformation.py
View file

@ -23,6 +23,7 @@ from litellm.types.containers.main import (
from litellm.types.router import GenericLiteLLMParams
from ...base_llm.containers.transformation import BaseContainerConfig
from ..workload_identity import resolve_openai_bearer_token
from .utils import join_container_api_base_path
if TYPE_CHECKING:
@ -104,13 +105,15 @@ class OpenAIContainerConfig(BaseContainerConfig):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
headers.update(
{
"Authorization": f"Bearer {api_key}",
},
params: Final = litellm_params or GenericLiteLLMParams()
bearer_token: Final = resolve_openai_bearer_token(
api_key=api_key or params.api_key,
api_base=params.api_base,
litellm_params=params.model_dump(exclude_none=True),
)
headers.update({"Authorization": f"Bearer {bearer_token}"})
return headers
def get_complete_url(

6
litellm/llms/openai/decisions/transformation.py
View file

@ -6,6 +6,7 @@ from typing_extensions import assert_never
import litellm
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text
from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
from litellm.secret_managers.main import get_secret_str
from litellm.types.decisions import (
DecisionsImage,
@ -349,6 +350,11 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
def resolve_api_key(self, api_key: str | None) -> str | None:
return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
return False

23
litellm/llms/openai/evals/transformation.py
View file

@ -13,6 +13,7 @@ from litellm.llms.base_llm.evals.transformation import (
BaseEvalsAPIConfig,
LiteLLMLoggingObj,
)
from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
CancelRunResponse,
@ -44,24 +45,14 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
return LlmProviders.OPENAI
def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
"""Add OpenAI-specific headers"""
import litellm
from litellm.secret_managers.main import get_secret_str
# Get API key following OpenAI pattern
api_key = None
if litellm_params:
api_key = litellm_params.api_key
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
if not api_key:
params: Final = litellm_params or GenericLiteLLMParams()
bearer_token: Final = resolve_openai_bearer_token(
api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
)
if not bearer_token:
raise ValueError("OPENAI_API_KEY is required for Evals API")
# Add required headers
headers["Authorization"] = f"Bearer {api_key}"
headers["Authorization"] = f"Bearer {bearer_token}"
headers["Content-Type"] = "application/json"
return headers
def get_complete_url(

50
litellm/llms/openai/fine_tuning/handler.py
View file

@ -6,7 +6,7 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
from openai.types.fine_tuning import FineTuningJob
from litellm._logging import verbose_logger
from litellm.llms.openai.common_utils import OpenAIAsyncHTTPClient, OpenAIHTTPClient
from litellm.llms.openai.workload_identity import build_async_openai_client, build_openai_client
from litellm.types.utils import LiteLLMFineTuningJob
_AZURE_STATUS_MAP: Final[Mapping[object, str]] = {
@ -74,27 +74,27 @@ class OpenAIFineTuningAPI:
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
_is_async: bool = False,
api_version: str | None = None,
litellm_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
received_args: Final = locals()
openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None
if client is None:
data: Final = {}
for k, v in received_args.items():
if k == "self" or k == "client" or k == "_is_async":
pass
elif k == "api_base" and v is not None:
data["base_url"] = v
elif v is not None:
data[k] = v
if _is_async is True:
openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
else:
openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
else:
openai_client = client
return openai_client
if client is not None:
return client
if _is_async:
return build_async_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
return build_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
async def acreate_fine_tuning_job(
self,
@ -116,6 +116,7 @@ class OpenAIFineTuningAPI:
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -126,6 +127,7 @@ class OpenAIFineTuningAPI:
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -164,6 +166,7 @@ class OpenAIFineTuningAPI:
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -174,6 +177,7 @@ class OpenAIFineTuningAPI:
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -212,6 +216,7 @@ class OpenAIFineTuningAPI:
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
after: str | None = None,
limit: int | None = None,
):
@ -224,6 +229,7 @@ class OpenAIFineTuningAPI:
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -263,6 +269,7 @@ class OpenAIFineTuningAPI:
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -273,6 +280,7 @@ class OpenAIFineTuningAPI:
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(

9
litellm/llms/openai/image_edit/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ImageResponse
from ..common_utils import OpenAIError
from ..workload_identity import resolve_openai_bearer_token
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -160,12 +161,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
litellm_params: dict | None = None,
api_base: str | None = None,
) -> dict:
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
bearer_token: Final = resolve_openai_bearer_token(
api_key=api_key, api_base=api_base, litellm_params=litellm_params
)
headers.update({"Authorization": f"Bearer {bearer_token}"})
return headers
def get_complete_url(

85
litellm/llms/openai/image_variations/handler.py
View file

@ -2,7 +2,7 @@
OpenAI Image Variations Handler
"""
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Final
import httpx
@ -14,27 +14,58 @@ from litellm.utils import ProviderConfigManager
from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
from ..common_utils import OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
from ..common_utils import OpenAIError
from ..workload_identity import build_async_openai_client, build_openai_client
class OpenAIImageVariationsHandler:
def get_sync_client(
self,
client: OpenAI | None,
init_client_params: dict,
):
api_key: str | None,
api_base: str,
timeout: float | httpx.Timeout | None,
max_retries: int,
organization: str | None,
litellm_params: Mapping[str, object],
) -> OpenAI:
if client is not None:
return client
return OpenAI(**init_client_params, http_client=litellm.client_session or OpenAIHTTPClient())
return build_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.client_session,
)
def get_async_client(self, client: AsyncOpenAI | None, init_client_params: dict) -> AsyncOpenAI:
def get_async_client(
self,
client: AsyncOpenAI | None,
api_key: str | None,
api_base: str,
timeout: float | httpx.Timeout | None,
max_retries: int,
organization: str | None,
litellm_params: Mapping[str, object],
) -> AsyncOpenAI:
if client is not None:
return client
return AsyncOpenAI(**init_client_params, http_client=litellm.aclient_session or OpenAIAsyncHTTPClient())
return build_async_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.aclient_session,
)
async def async_image_variations(
self,
api_key: str,
api_key: str | None,
api_base: str,
organization: str | None,
client: AsyncOpenAI | None,
@ -51,15 +82,15 @@ class OpenAIImageVariationsHandler:
provider_config: BaseImageVariationConfig,
) -> ImageResponse:
try:
init_client_params: Final = {
"api_key": api_key,
"base_url": api_base,
"timeout": timeout,
"max_retries": max_retries,
"organization": organization,
}
client = self.get_async_client(client=client, init_client_params=init_client_params)
client = self.get_async_client(
client=client,
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
raw_response: Final = await client.images.with_raw_response.create_variation(**data)
response: Final = raw_response.parse()
@ -103,7 +134,7 @@ class OpenAIImageVariationsHandler:
def image_variations(
self,
model_response: ImageResponse,
api_key: str,
api_key: str | None,
api_base: str,
model: str | None,
image: FileTypes,
@ -167,15 +198,15 @@ class OpenAIImageVariationsHandler:
litellm_params=litellm_params,
)
init_client_params: Final = {
"api_key": api_key,
"base_url": api_base,
"timeout": timeout,
"max_retries": max_retries,
"organization": organization,
}
client = self.get_sync_client(client=client, init_client_params=init_client_params)
client = self.get_sync_client(
client=client,
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
raw_response: Final = client.images.with_raw_response.create_variation(**json_data)
response: Final = raw_response.parse()

258
litellm/llms/openai/openai.py
View file

@ -55,16 +55,19 @@ from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig
from .chat.o_series_transformation import OpenAIOSeriesConfig
from .common_utils import (
BaseOpenAILLM,
OpenAIAsyncHTTPClient,
OpenAIError,
OpenAIHTTPClient,
build_output_token_limit_response,
drop_params_from_unprocessable_entity_error,
is_openai_backed_api_base,
is_output_token_limit_error,
model_dump_provider_response,
)
from .workload_identity import resolve_openai_workload_identity_config
from .workload_identity import (
build_async_openai_client,
build_openai_client,
openai_static_api_key,
resolve_openai_workload_identity_config,
)
openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
openAIGPT5Config: Final = OpenAIGPT5Config()
@ -441,7 +444,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
)
if workload_identity_config is not None
else AsyncOpenAI(
api_key=api_key,
api_key=openai_static_api_key(api_key),
base_url=api_base,
http_client=async_http_client,
timeout=timeout,
@ -463,7 +466,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
)
if workload_identity_config is not None
else OpenAI(
api_key=api_key,
api_key=openai_static_api_key(api_key),
base_url=api_base,
http_client=sync_http_client,
timeout=timeout,
@ -1491,10 +1494,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries=None,
organization: str | None = None,
headers: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
):
try:
openai_aclient: Final = self._get_openai_client(
is_async=True,
litellm_params=litellm_params,
api_key=api_key,
api_base=api_base,
timeout=timeout,
@ -1556,6 +1561,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
aimg_generation=None,
organization: str | None = None,
headers: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> ImageResponse:
data = {}
try:
@ -1577,10 +1583,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries=max_retries,
organization=organization,
headers=headers,
litellm_params=litellm_params,
)
openai_client: Final[OpenAI] = self._get_openai_client(
is_async=False,
litellm_params=litellm_params,
api_key=api_key,
api_base=api_base,
timeout=timeout,
@ -1658,6 +1666,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
aspeech: bool | None = None,
client=None,
shared_session: Optional["ClientSession"] = None,
litellm_params: Mapping[str, object] | None = None,
) -> HttpxBinaryResponseContent:
if aspeech is not None and aspeech is True:
return self.async_audio_speech(
@ -1674,6 +1683,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj=logging_obj,
client=client,
shared_session=shared_session,
litellm_params=litellm_params,
)
openai_client: Final = self._get_openai_client(
@ -1684,6 +1694,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries=max_retries,
client=client,
shared_session=shared_session,
litellm_params=litellm_params,
)
sync_client: Final = cast(OpenAI, openai_client)
@ -1721,6 +1732,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj: LiteLLMLoggingObj,
client=None,
shared_session: Optional["ClientSession"] = None,
litellm_params: Mapping[str, object] | None = None,
) -> HttpxBinaryResponseContent:
openai_client: Final = cast(
AsyncOpenAI,
@ -1732,6 +1744,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries=max_retries,
client=client,
shared_session=shared_session,
litellm_params=litellm_params,
),
)
@ -1778,26 +1791,27 @@ class OpenAIFilesAPI(BaseLLM):
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
_is_async: bool = False,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | None:
received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
openai_client: OpenAI | AsyncOpenAI | None = None
if client is None:
data: Final = {}
for k, v in received_args.items():
if k == "self" or k == "client" or k == "_is_async":
pass
elif k == "api_base" and v is not None:
data["base_url"] = v
elif v is not None:
data[k] = v
if _is_async is True:
openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
else:
openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
else:
openai_client = client
return openai_client
if client is not None:
return client
if _is_async:
return build_async_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
return build_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
async def acreate_file(
self,
@ -1817,6 +1831,7 @@ class OpenAIFilesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAIFileObject | Coroutine[None, None, OpenAIFileObject]:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -1826,6 +1841,7 @@ class OpenAIFilesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -1859,6 +1875,7 @@ class OpenAIFilesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -1868,6 +1885,7 @@ class OpenAIFilesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -1924,6 +1942,7 @@ class OpenAIFilesAPI(BaseLLM):
organization: str | None,
chunk_size: int = 1024 * 1024,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> FileContentStreamingResult:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -1933,6 +1952,7 @@ class OpenAIFilesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -1987,6 +2007,7 @@ class OpenAIFilesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -1996,6 +2017,7 @@ class OpenAIFilesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2033,6 +2055,7 @@ class OpenAIFilesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -2042,6 +2065,7 @@ class OpenAIFilesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2082,6 +2106,7 @@ class OpenAIFilesAPI(BaseLLM):
organization: str | None,
purpose: str | None = None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -2091,6 +2116,7 @@ class OpenAIFilesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2136,26 +2162,27 @@ class OpenAIBatchesAPI(BaseLLM):
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
_is_async: bool = False,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | None:
received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
openai_client: OpenAI | AsyncOpenAI | None = None
if client is None:
data: Final = {}
for k, v in received_args.items():
if k == "self" or k == "client" or k == "_is_async":
pass
elif k == "api_base" and v is not None:
data["base_url"] = v
elif v is not None:
data[k] = v
if _is_async is True:
openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
else:
openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
else:
openai_client = client
return openai_client
if client is not None:
return client
if _is_async:
return build_async_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
return build_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
async def acreate_batch(
self,
@ -2175,6 +2202,7 @@ class OpenAIBatchesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -2184,6 +2212,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2219,6 +2248,7 @@ class OpenAIBatchesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -2228,6 +2258,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2262,6 +2293,7 @@ class OpenAIBatchesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -2271,6 +2303,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2311,6 +2344,7 @@ class OpenAIBatchesAPI(BaseLLM):
after: str | None = None,
limit: int | None = None,
client: OpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -2320,6 +2354,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization=organization,
client=client,
_is_async=_is_async,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -2348,22 +2383,19 @@ class OpenAIAssistantsAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI:
received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
if client is None:
data: Final = {}
for k, v in received_args.items():
if k == "self" or k == "client":
pass
elif k == "api_base" and v is not None:
data["base_url"] = v
elif v is not None:
data[k] = v
openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
else:
openai_client = client
return openai_client
if client is not None:
return client
return build_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
def async_get_openai_client(
self,
@ -2373,22 +2405,19 @@ class OpenAIAssistantsAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: AsyncOpenAI | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> AsyncOpenAI:
received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
if client is None:
data: Final = {}
for k, v in received_args.items():
if k == "self" or k == "client":
pass
elif k == "api_base" and v is not None:
data["base_url"] = v
elif v is not None:
data[k] = v
openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
else:
openai_client = client
return openai_client
if client is not None:
return client
return build_async_openai_client(
api_key=api_key,
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
)
### ASSISTANTS ###
@ -2404,6 +2433,8 @@ class OpenAIAssistantsAPI(BaseLLM):
limit: int | None = 20,
before: str | None = None,
after: str | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> AsyncCursorPage[Assistant]:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2411,6 +2442,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
request_params: Final = {
@ -2438,6 +2470,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
aget_assistants: Literal[True],
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, AsyncCursorPage[Assistant]]:
...
@ -2451,6 +2485,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: OpenAI | None,
aget_assistants: Literal[False] | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> SyncCursorPage[Assistant]:
...
@ -2469,7 +2505,9 @@ class OpenAIAssistantsAPI(BaseLLM):
limit: int | None = 20,
before: str | None = None,
after: str | None = None,
):
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, AsyncCursorPage[Assistant]] | SyncCursorPage[Assistant]:
if aget_assistants is not None and aget_assistants is True:
return self.async_get_assistants(
api_key=api_key,
@ -2477,6 +2515,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
openai_client: Final = self.get_openai_client(
@ -2485,6 +2524,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2512,6 +2552,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
create_assistant_data: dict,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Assistant:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2519,6 +2561,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2536,6 +2579,8 @@ class OpenAIAssistantsAPI(BaseLLM):
create_assistant_data: dict,
client=None,
async_create_assistants=None,
*,
litellm_params: Mapping[str, object] | None = None,
):
if async_create_assistants is not None and async_create_assistants is True:
return self.async_create_assistants(
@ -2544,6 +2589,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
create_assistant_data=create_assistant_data,
)
@ -2553,6 +2599,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2569,6 +2616,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
assistant_id: str,
*,
litellm_params: Mapping[str, object] | None = None,
) -> AssistantDeleted:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2576,6 +2625,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2593,6 +2643,8 @@ class OpenAIAssistantsAPI(BaseLLM):
assistant_id: str,
client=None,
async_delete_assistants=None,
*,
litellm_params: Mapping[str, object] | None = None,
):
if async_delete_assistants is not None and async_delete_assistants is True:
return self.async_delete_assistant(
@ -2601,6 +2653,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
assistant_id=assistant_id,
)
@ -2610,6 +2663,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2628,6 +2682,8 @@ class OpenAIAssistantsAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: AsyncOpenAI | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAIMessage:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2635,6 +2691,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2665,6 +2722,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
a_add_message: Literal[True],
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, OpenAIMessage]:
...
@ -2680,6 +2739,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: OpenAI | None,
a_add_message: Literal[False] | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAIMessage:
...
@ -2696,6 +2757,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client=None,
a_add_message: bool | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
):
if a_add_message is not None and a_add_message is True:
return self.a_add_message(
@ -2706,6 +2769,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
openai_client: Final = self.get_openai_client(
@ -2714,6 +2778,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2739,6 +2804,8 @@ class OpenAIAssistantsAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: AsyncOpenAI | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> AsyncCursorPage[OpenAIMessage]:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2746,6 +2813,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2766,6 +2834,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
aget_messages: Literal[True],
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
...
@ -2780,6 +2850,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: OpenAI | None,
aget_messages: Literal[False] | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> SyncCursorPage[OpenAIMessage]:
...
@ -2795,6 +2867,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client=None,
aget_messages=None,
*,
litellm_params: Mapping[str, object] | None = None,
):
if aget_messages is not None and aget_messages is True:
return self.async_get_messages(
@ -2804,6 +2878,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
openai_client: Final = self.get_openai_client(
@ -2812,6 +2887,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2831,6 +2907,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
messages: Iterable[OpenAICreateThreadParamsMessage] | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Thread:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2838,6 +2916,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2870,6 +2949,8 @@ class OpenAIAssistantsAPI(BaseLLM):
messages: Iterable[OpenAICreateThreadParamsMessage] | None,
client: AsyncOpenAI | None,
acreate_thread: Literal[True],
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, Thread]:
...
@ -2885,6 +2966,8 @@ class OpenAIAssistantsAPI(BaseLLM):
messages: Iterable[OpenAICreateThreadParamsMessage] | None,
client: OpenAI | None,
acreate_thread: Literal[False] | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Thread:
...
@ -2901,6 +2984,8 @@ class OpenAIAssistantsAPI(BaseLLM):
messages: Iterable[OpenAICreateThreadParamsMessage] | None,
client=None,
acreate_thread=None,
*,
litellm_params: Mapping[str, object] | None = None,
):
"""
Here's an example:
@ -2920,6 +3005,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
messages=messages,
)
@ -2929,6 +3015,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2956,6 +3043,8 @@ class OpenAIAssistantsAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: AsyncOpenAI | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Thread:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -2963,6 +3052,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -2988,6 +3078,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None,
aget_thread: Literal[True],
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, Thread]:
...
@ -3002,6 +3094,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: OpenAI | None,
aget_thread: Literal[False] | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Thread:
...
@ -3017,6 +3111,8 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client=None,
aget_thread=None,
*,
litellm_params: Mapping[str, object] | None = None,
):
if aget_thread is not None and aget_thread is True:
return self.async_get_thread(
@ -3026,6 +3122,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
openai_client: Final = self.get_openai_client(
@ -3034,6 +3131,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -3067,6 +3165,8 @@ class OpenAIAssistantsAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: AsyncOpenAI | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Run:
openai_client: Final = self.async_get_openai_client(
api_key=api_key,
@ -3074,6 +3174,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
@ -3178,6 +3279,8 @@ class OpenAIAssistantsAPI(BaseLLM):
client,
arun_thread: Literal[True],
event_handler: AssistantEventHandler | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Coroutine[None, None, Run]:
...
@ -3200,6 +3303,8 @@ class OpenAIAssistantsAPI(BaseLLM):
client,
arun_thread: Literal[False] | None,
event_handler: AssistantEventHandler | None,
*,
litellm_params: Mapping[str, object] | None = None,
) -> Run:
...
@ -3223,6 +3328,8 @@ class OpenAIAssistantsAPI(BaseLLM):
client=None,
arun_thread=None,
event_handler: AssistantEventHandler | None = None,
*,
litellm_params: Mapping[str, object] | None = None,
):
if arun_thread is not None and arun_thread is True:
if stream is not None and stream is True:
@ -3232,6 +3339,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
return self.async_run_thread_stream(
@ -3259,6 +3367,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)
openai_client: Final = self.get_openai_client(
@ -3267,6 +3376,7 @@ class OpenAIAssistantsAPI(BaseLLM):
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
client=client,
)

31
litellm/llms/openai/responses/count_tokens/token_counter.py
View file

@ -27,6 +27,7 @@ from litellm.llms.openai.responses.count_tokens.transformation import (
from litellm.llms.openai.workload_identity import (
OpenAIWorkloadIdentityConfig,
get_workload_identity_bearer_token,
openai_static_api_key,
resolve_openai_workload_identity_config,
)
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
@ -35,13 +36,18 @@ from litellm.types.utils import CountContextManagement, LlmProviders, TokenCount
openai_count_tokens_handler: Final = OpenAICountTokensHandler()
def _workload_identity_config(litellm_params: Mapping[str, object]) -> OpenAIWorkloadIdentityConfig | None:
def _workload_identity_config(
litellm_params: Mapping[str, object],
) -> OpenAIWorkloadIdentityConfig | OpenAIError | None:
api_base: Final = litellm_params.get("api_base")
return resolve_openai_workload_identity_config(
api_key=None,
api_base=api_base if isinstance(api_base, str) else None,
litellm_params=litellm_params,
)
try:
return resolve_openai_workload_identity_config(
api_key=None,
api_base=api_base if isinstance(api_base, str) else None,
litellm_params=litellm_params,
)
except OpenAIError as e:
return e
class OpenAITokenCounter(BaseTokenCounter):
@ -127,7 +133,18 @@ class OpenAITokenCounter(BaseTokenCounter):
credentials: Final = get_openai_credentials(
api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key")
)
credential: Final = credentials.api_key or _workload_identity_config(litellm_params)
credential: Final = openai_static_api_key(credentials.api_key) or _workload_identity_config(litellm_params)
if isinstance(credential, OpenAIError):
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="openai_api",
error=True,
error_message=credential.message,
status_code=credential.status_code,
)
if credential is None:
return TokenCountResponse(

5
litellm/llms/openai/transcriptions/handler.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Optional, cast
import httpx
@ -97,6 +98,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
max_retries=max_retries,
logging_obj=logging_obj,
shared_session=shared_session,
litellm_params=litellm_params,
)
openai_client: Final[OpenAI] = self._get_openai_client(
@ -106,6 +108,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
timeout=timeout,
max_retries=max_retries,
client=client,
litellm_params=litellm_params,
)
## LOGGING
@ -159,6 +162,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
client=None,
max_retries=None,
shared_session: Optional["ClientSession"] = None,
litellm_params: Mapping[str, object] | None = None,
):
try:
openai_aclient: Final[AsyncOpenAI] = self._get_openai_client(
@ -169,6 +173,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
max_retries=max_retries,
client=client,
shared_session=shared_session,
litellm_params=litellm_params,
)
## LOGGING

9
litellm/llms/openai/vector_store_files/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store_files.transformation import (
BaseVectorStoreFilesConfig,
)
from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.vector_store_files import (
@ -70,11 +71,13 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
headers: dict[str, str],
litellm_params: GenericLiteLLMParams | None,
) -> dict[str, str]:
litellm_params = litellm_params or GenericLiteLLMParams()
api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
params: Final = litellm_params or GenericLiteLLMParams()
bearer_token: Final = resolve_openai_bearer_token(
api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
)
headers.update(
{
"Authorization": f"Bearer {api_key}",
"Authorization": f"Bearer {bearer_token}",
"Content-Type": "application/json",
}
)

9
litellm/llms/openai/vector_stores/transformation.py
View file

@ -5,6 +5,7 @@ import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.vector_stores import (
@ -48,11 +49,13 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
}
def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
params: Final = litellm_params or GenericLiteLLMParams()
bearer_token: Final = resolve_openai_bearer_token(
api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
)
headers.update(
{
"Authorization": f"Bearer {api_key}",
"Authorization": f"Bearer {bearer_token}",
"Content-Type": "application/json",
}
)

18
litellm/llms/openai/videos/transformation.py
View file

@ -8,11 +8,10 @@ from urllib.parse import quote
import httpx
from httpx._types import FileContent, FileTypes, RequestFiles
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils
from litellm.secret_managers.main import get_secret_str
from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
from litellm.types.llms.openai import CreateVideoRequest
from litellm.types.router import GenericLiteLLMParams
from litellm.types.videos.main import (
@ -78,16 +77,13 @@ class OpenAIVideoConfig(BaseVideoConfig):
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
# Use api_key from litellm_params if available, otherwise fall back to other sources
if litellm_params and litellm_params.api_key:
api_key = api_key or litellm_params.api_key
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
params: Final = litellm_params or GenericLiteLLMParams()
bearer_token: Final = resolve_openai_bearer_token(
api_key=api_key or params.api_key,
api_base=params.api_base,
litellm_params=params.model_dump(exclude_none=True),
)
headers.update({"Authorization": f"Bearer {bearer_token}"})
return headers
def get_complete_url(

246
litellm/llms/openai/workload_identity.py
View file

@ -6,10 +6,22 @@ from functools import lru_cache
from typing import TYPE_CHECKING, Final
from urllib.parse import urlparse
import litellm
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
import httpx
from openai import DEFAULT_MAX_RETRIES as OPENAI_SDK_DEFAULT_MAX_RETRIES
from openai import NOT_GIVEN, AsyncOpenAI, NotGiven, OpenAI
from .common_utils import OpenAIError, is_openai_backed_api_base
import litellm
from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
from .common_utils import (
BaseOpenAILLM,
OpenAIAsyncHTTPClient,
OpenAIError,
OpenAIHTTPClient,
is_openai_backed_api_base,
)
if TYPE_CHECKING:
from collections.abc import Callable
@ -17,6 +29,11 @@ if TYPE_CHECKING:
from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
OPENAI_WIF_CLIENT_ID: Final = "litellm"
_IDENTITY_PARAM_ENV_NAMES: Final = (
("openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"),
("openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID"),
("openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE"),
)
_SDK_UPGRADE_MESSAGE: Final = (
"OpenAI workload identity federation requires openai>=2.32.0. "
"Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
@ -51,17 +68,25 @@ def resolve_openai_workload_identity_config(
)
if static_api_key is not None:
return None
effective_api_base: Final = (
api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
)
if not _targets_openai_api(effective_api_base):
return _configured_openai_workload_identity(api_base=api_base, litellm_params=litellm_params)
def _effective_openai_api_base(api_base: str | None) -> str | None:
return api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
def _configured_openai_workload_identity(
api_base: str | None,
litellm_params: Mapping[str, object] | None,
) -> OpenAIWorkloadIdentityConfig | None:
if not _targets_openai_api(_effective_openai_api_base(api_base)):
return None
identity_provider_id: Final = _config_value(
litellm_params, "openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"
identity_provider_id, service_account_id, token_file = (
_config_value(litellm_params, param_key, env_name) for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES
)
service_account_id: Final = _config_value(litellm_params, "openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID")
token_file: Final = _config_value(litellm_params, "openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE")
if identity_provider_id is None or service_account_id is None or token_file is None:
if _carries_deployment_identity(litellm_params):
raise OpenAIError(status_code=401, message=_incomplete_identity_message(litellm_params))
return None
return OpenAIWorkloadIdentityConfig(
identity_provider_id=identity_provider_id,
@ -81,11 +106,200 @@ async def get_workload_identity_bearer_token_for_api_base(api_base: str) -> str
return await _workload_identity_auth(config).get_token_async()
def openai_static_api_key(api_key: str | None) -> str | None:
return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
def resolve_openai_bearer_token(
api_key: str | None,
api_base: str | None,
litellm_params: Mapping[str, object] | None,
) -> str | None:
static_api_key: Final = openai_static_api_key(api_key)
workload_identity_config: Final = resolve_openai_workload_identity_config(
api_key=static_api_key, api_base=api_base, litellm_params=litellm_params
)
if workload_identity_config is None:
return static_api_key
return get_workload_identity_bearer_token(workload_identity_config)
async def resolve_openai_bearer_token_async(
static_api_key: str | None,
api_base: str | None,
litellm_params: Mapping[str, object] | None,
) -> str | None:
workload_identity_config: Final = resolve_openai_workload_identity_config(
api_key=static_api_key, api_base=api_base, litellm_params=litellm_params
)
if workload_identity_config is None:
return static_api_key
return await _workload_identity_auth(workload_identity_config).get_token_async()
def _carries_deployment_identity(litellm_params: Mapping[str, object] | None) -> bool:
if litellm_params is None:
return False
return any(_param_str(litellm_params, key) is not None for key in OPENAI_WIF_KWARGS_KEYS)
def _incomplete_identity_message(litellm_params: Mapping[str, object] | None) -> str:
missing: Final = ", ".join(
param_key
for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES
if _config_value(litellm_params, param_key, env_name) is None
)
return (
f"OpenAI workload identity federation on this deployment is missing {missing}. "
"Set openai_identity_provider_id, openai_service_account_id, and openai_identity_token_file "
"on the deployment, or the OPENAI_IDENTITY_PROVIDER_ID / OPENAI_SERVICE_ACCOUNT_ID / "
"OPENAI_IDENTITY_TOKEN_FILE env vars they fall back to."
)
def _param_str(litellm_params: Mapping[str, object], key: str) -> str | None:
value: Final = litellm_params.get(key)
return value if isinstance(value, str) and value else None
def _config_value(litellm_params: Mapping[str, object] | None, param_key: str, env_name: str) -> str | None:
param_value: Final = litellm_params.get(param_key) if litellm_params is not None else None
if isinstance(param_value, str) and param_value:
return param_value
return normalize_nonempty_secret_str(get_secret_str(env_name))
param_value: Final = _param_str(litellm_params, param_key) if litellm_params is not None else None
return param_value or normalize_nonempty_secret_str(get_secret_str(env_name))
def build_openai_client(
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
max_retries: int | None = None,
organization: str | None = None,
litellm_params: Mapping[str, object] | None = None,
static_key_http_client_factory: Callable[[], httpx.Client | None] | None = None,
) -> OpenAI:
workload_identity_config: Final = resolve_openai_workload_identity_config(
api_key=api_key, api_base=api_base, litellm_params=litellm_params
)
retries: Final = _max_retries_or_sdk_default(max_retries)
if workload_identity_config is None:
static_key_http_client: Final = (
None if static_key_http_client_factory is None else static_key_http_client_factory()
)
return OpenAI(
api_key=openai_static_api_key(api_key),
base_url=api_base,
timeout=timeout,
max_retries=retries,
organization=organization,
http_client=static_key_http_client or OpenAIHTTPClient(),
)
cache_params: Final = _client_cache_params(
is_async=False,
workload_identity_config=workload_identity_config,
api_base=api_base,
timeout=timeout,
max_retries=retries,
organization=organization,
)
cached: Final = BaseOpenAILLM.get_cached_openai_client(
client_initialization_params=cache_params, client_type="openai"
)
if isinstance(cached, OpenAI):
return cached
process_http_client: Final = BaseOpenAILLM.get_sync_http_client()
client: Final = OpenAI(
workload_identity=workload_identity_config.to_sdk_workload_identity(),
base_url=api_base,
timeout=timeout,
max_retries=retries,
organization=organization,
http_client=process_http_client,
)
BaseOpenAILLM.set_cached_openai_client(
openai_client=client,
client_type="openai",
client_initialization_params=cache_params,
litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client),
)
return client
def build_async_openai_client(
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
max_retries: int | None = None,
organization: str | None = None,
litellm_params: Mapping[str, object] | None = None,
static_key_http_client_factory: Callable[[], httpx.AsyncClient | None] | None = None,
) -> AsyncOpenAI:
workload_identity_config: Final = resolve_openai_workload_identity_config(
api_key=api_key, api_base=api_base, litellm_params=litellm_params
)
retries: Final = _max_retries_or_sdk_default(max_retries)
if workload_identity_config is None:
static_key_http_client: Final = (
None if static_key_http_client_factory is None else static_key_http_client_factory()
)
return AsyncOpenAI(
api_key=openai_static_api_key(api_key),
base_url=api_base,
timeout=timeout,
max_retries=retries,
organization=organization,
http_client=static_key_http_client or OpenAIAsyncHTTPClient(),
)
cache_params: Final = _client_cache_params(
is_async=True,
workload_identity_config=workload_identity_config,
api_base=api_base,
timeout=timeout,
max_retries=retries,
organization=organization,
)
cached: Final = BaseOpenAILLM.get_cached_openai_client(
client_initialization_params=cache_params, client_type="openai"
)
if isinstance(cached, AsyncOpenAI):
return cached
process_http_client: Final = BaseOpenAILLM.get_async_http_client()
client: Final = AsyncOpenAI(
workload_identity=workload_identity_config.to_sdk_workload_identity(),
base_url=api_base,
timeout=timeout,
max_retries=retries,
organization=organization,
http_client=process_http_client,
)
BaseOpenAILLM.set_cached_openai_client(
openai_client=client,
client_type="openai",
client_initialization_params=cache_params,
litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client),
)
return client
def _client_cache_params(
is_async: bool,
workload_identity_config: OpenAIWorkloadIdentityConfig,
api_base: str | None,
timeout: float | httpx.Timeout | None | NotGiven,
max_retries: int,
organization: str | None,
) -> dict[str, object]:
return {
"api_key": None,
"is_async": is_async,
"workload_identity_config": workload_identity_config,
"api_base": api_base,
"timeout": timeout,
"max_retries": max_retries,
"organization": organization,
}
def _max_retries_or_sdk_default(max_retries: int | None) -> int:
return OPENAI_SDK_DEFAULT_MAX_RETRIES if max_retries is None else max_retries
def _targets_openai_api(api_base: str | None) -> bool:
@ -95,7 +309,7 @@ def _targets_openai_api(api_base: str | None) -> bool:
return parsed.scheme == "https" and is_openai_backed_api_base(api_base)
@lru_cache(maxsize=16)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth:
sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth()
return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity())

31
litellm/main.py
View file

@ -263,6 +263,7 @@ from .llms.openai.completion.handler import OpenAITextCompletion
from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler
from .llms.openai.openai import OpenAIChatCompletion
from .llms.openai.transcriptions.handler import OpenAIAudioTranscription
from .llms.openai.workload_identity import build_openai_client
from .llms.openai_like.chat.handler import OpenAILikeChatHandler
from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler
from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
@ -7954,17 +7955,14 @@ def adapter_completion(*, adapter_id: str, **kwargs) -> BaseModel | AdapterCompl
def moderation(input: str, model: str | None = None, api_key: str | None = None, **kwargs) -> OpenAIModerationResponse:
from litellm.llms.openai.common_utils import OpenAIHTTPClient
# only supports open ai for now
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
# Extract api_base from kwargs
static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
api_base: Final = kwargs.get("api_base", None)
openai_client = kwargs.get("client", None)
if openai_client is None:
openai_client = openai.OpenAI(api_key=api_key, base_url=api_base, http_client=OpenAIHTTPClient())
openai_client: Final = kwargs.get("client", None) or build_openai_client(
api_key=static_api_key,
api_base=api_base,
litellm_params=kwargs,
)
if model is not None:
response = openai_client.moderations.create(input=input, model=model)
@ -7987,8 +7985,7 @@ async def amoderation(
) -> OpenAIModerationResponse:
from openai import AsyncOpenAI
# only supports open ai for now
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
optional_params: Final = GenericLiteLLMParams.model_validate(kwargs)
litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
_dynamic_api_base = None
@ -8014,8 +8011,9 @@ async def amoderation(
# _get_openai_client maintains in-memory caching logic for OpenAI clients
_openai_client: AsyncOpenAI = openai_chat_completions.get_openai_client(
is_async=True,
api_key=api_key,
api_key=static_api_key,
api_base=optional_params.api_base or _dynamic_api_base,
litellm_params=optional_params.model_dump(exclude_none=True),
)
else:
_openai_client = openai_client
@ -8035,7 +8033,7 @@ async def amoderation(
moderation_request: Final = {"input": input, "model": model}
litellm_logging_obj.pre_call(
input=input,
api_key=api_key,
api_key=static_api_key,
additional_args={
"complete_input_dict": moderation_request,
"api_base": str(_openai_client.base_url),
@ -8268,9 +8266,7 @@ def transcription(
or get_secret("OPENAI_ORGANIZATION")
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
response = openai_audio_transcriptions.audio_transcriptions(
model=model,
audio_file=file,
@ -8282,7 +8278,7 @@ def transcription(
logging_obj=litellm_logging_obj,
max_retries=max_retries,
api_base=api_base,
api_key=api_key,
api_key=static_api_key,
provider_config=provider_config,
litellm_params=litellm_params_dict,
shared_session=shared_session,
@ -8604,6 +8600,7 @@ def speech(
client=client, # pass AsyncOpenAI, OpenAI client
aspeech=aspeech,
shared_session=shared_session,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
# Check if this is Azure Speech Service (Cognitive Services TTS)

29
litellm/proxy/auth/auth_utils.py
View file

@ -553,6 +553,29 @@ def _iter_fallback_targets(value: object, depth: int) -> Iterator[str | Mapping[
yield from _iter_fallback_targets(target_list, depth + 1)
def _reject_client_chosen_federation(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None:
reject_server_owned_wif_params(litellm_params)
if not manages_deployments:
reject_federated_credential_reference(litellm_params)
def _iter_user_config_litellm_params(user_config: object) -> Iterator[Mapping[str, object]]:
config: Final = _coerce_metadata_to_dict(user_config)
if config is None:
return
if (defaults := _coerce_metadata_to_dict(config.get("default_litellm_params"))) is not None:
yield defaults
model_list: Final = config.get("model_list")
if not isinstance(model_list, list):
return
for deployment in model_list:
if (
isinstance(deployment, Mapping)
and (params := _coerce_metadata_to_dict(deployment.get("litellm_params"))) is not None
):
yield params
def iter_request_fallback_targets(request_body: Mapping[str, object]) -> Iterator[str | Mapping[str, object]]:
for value in _iter_fallback_field_values(request_body):
yield from _iter_fallback_targets(value, 0)
@ -685,11 +708,11 @@ def is_request_body_safe(
_reject_url_valued_fallback_target(target_model)
elif isinstance(target, str):
_reject_url_valued_fallback_target(target)
for user_config_params in _iter_user_config_litellm_params(request_body.get("user_config")):
_reject_client_chosen_federation(user_config_params, manages_deployments=manages_deployments)
litellm_params: Final = _coerce_metadata_to_dict(request_body.get("litellm_params"))
if litellm_params is not None:
reject_server_owned_wif_params(litellm_params)
if not manages_deployments:
reject_federated_credential_reference(litellm_params)
_reject_client_chosen_federation(litellm_params, manages_deployments=manages_deployments)
litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata"))
if litellm_params_metadata is not None:
_check_banned_params(

3
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -17,6 +17,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest
from litellm.litellm_core_utils.hidden_params import get_hidden_params, set_hidden_param
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
@ -171,6 +172,8 @@ def _raise_not_found_when_openai_fallback_unservable(
return
if data.get("api_key") or litellm.api_key or litellm.openai_key or os.getenv("OPENAI_API_KEY"):
return
if resolve_openai_workload_identity_config(api_key=None, api_base=None, litellm_params=data) is not None:
return
raise ProxyException(
message=not_found_message,
type="invalid_request_error",

2
litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py
View file

@ -5,6 +5,7 @@ from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import (
OpenAIModerationGuardrail,
)
from litellm.types.guardrails import SupportedGuardrailIntegrations
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
if TYPE_CHECKING:
from litellm.types.guardrails import Guardrail, LitellmParams
@ -19,6 +20,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
openai_moderation_guardrail: Final = OpenAIModerationGuardrail(
guardrail_name=guardrail_name,
workload_identity_params=litellm_params.model_dump(include=set(OPENAI_WIF_KWARGS_KEYS), exclude_none=True),
**{
**litellm_params.model_dump(exclude_none=True),
"api_key": litellm_params.api_key,

36
litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py
View file

@ -3,6 +3,8 @@
OpenAI Moderation Guardrail Integration for LiteLLM
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Optional
from fastapi import HTTPException
@ -17,7 +19,14 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.workload_identity import (
OpenAIWorkloadIdentityConfig,
resolve_openai_bearer_token_async,
resolve_openai_workload_identity_config,
)
from litellm.proxy.common_utils.callback_utils import add_guardrail_scan_id
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations
from litellm.types.utils import (
GenericGuardrailAPIInputs,
@ -32,6 +41,21 @@ if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
def _config_secret(value: object) -> object:
return get_secret_str(value) if isinstance(value, str) and value.startswith("os.environ/") else value
def _guardrail_workload_identity(
api_base: str, workload_identity_params: Mapping[str, object]
) -> OpenAIWorkloadIdentityConfig | None:
try:
return resolve_openai_workload_identity_config(
api_key=None, api_base=api_base, litellm_params=workload_identity_params
)
except OpenAIError as incomplete_identity:
raise ValueError(f"OpenAI Moderation: {incomplete_identity.message}") from incomplete_identity
class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
"""
LiteLLM Built-in Guardrail for OpenAI Content Moderation.
@ -55,6 +79,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
model: Literal["omni-moderation-latest", "text-moderation-latest"] | None = None,
streaming_end_of_stream_only: bool | None = None,
streaming_sampling_rate: int | None = None,
workload_identity_params: Mapping[str, object] | None = None,
**kwargs,
):
"""Initialize OpenAI Moderation guardrail handler."""
@ -78,7 +103,11 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
)
self.streaming_sampling_rate: int = 5 if streaming_sampling_rate is None else streaming_sampling_rate
if not self.api_key:
self.workload_identity_params: Mapping[str, object] = MappingProxyType(
{key: _config_secret(value) for key, value in (workload_identity_params or {}).items()}
)
if not self.api_key and _guardrail_workload_identity(self.api_base, self.workload_identity_params) is None:
raise ValueError(
"OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration."
)
@ -109,10 +138,13 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
verbose_proxy_logger.debug("OpenAI Moderation guard request: %s", request_body)
bearer_token: Final = await resolve_openai_bearer_token_async(
static_api_key=self.api_key, api_base=self.api_base, litellm_params=self.workload_identity_params
)
response: Final = await self.async_handler.post(
url=f"{self.api_base}/moderations",
headers={
"Authorization": f"Bearer {self.api_key}",
"Authorization": f"Bearer {bearer_token}",
"Content-Type": "application/json",
},
json=request_body,

45
litellm/realtime_api/main.py
View file

@ -44,6 +44,7 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_co
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
from ..llms.openai.live.handler import OpenAILiveSessions
from ..llms.openai.realtime.handler import OpenAIRealtime
from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
from ..llms.vertex_ai.vertex_llm_base import VertexBase
@ -131,6 +132,17 @@ def _get_realtime_http_provider_config(
return provider_config, resolved_api_base.rstrip("/"), resolved_api_key
async def _realtime_http_bearer_token(
custom_llm_provider: str, api_key: str, api_base: str, litellm_params: GenericLiteLLMParams
) -> str:
if custom_llm_provider != LlmProviders.OPENAI.value:
return api_key
bearer_token: Final = await resolve_openai_bearer_token_async(
static_api_key=api_key or None, api_base=api_base, litellm_params=litellm_params.model_dump(exclude_none=True)
)
return bearer_token or ""
@wrapper_client
async def acreate_realtime_client_secret(
model: str | None = None,
@ -181,7 +193,9 @@ async def acreate_realtime_client_secret(
request_data["session"] = _with_resolved_session_model(request_data["session"], model_name)
return await base_llm_http_handler.async_realtime_client_secret_handler(
api_base=resolved_api_base,
api_key=resolved_api_key,
api_key=await _realtime_http_bearer_token(
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
),
request_data=request_data,
logging_obj=litellm_logging_obj,
timeout=timeout or request_timeout,
@ -254,7 +268,9 @@ async def acreate_realtime_transcription_session(
request_data["input_audio_transcription"]["model"] = model_name
return await base_llm_http_handler.async_realtime_transcription_session_handler(
api_base=resolved_api_base,
api_key=resolved_api_key,
api_key=await _realtime_http_bearer_token(
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
),
request_data=request_data,
logging_obj=litellm_logging_obj,
timeout=timeout or request_timeout,
@ -419,17 +435,21 @@ async def _alive_session(
api_base: str | None,
api_key: str | None,
litellm_metadata: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> None:
rejection: Final = _live_session_rejection(custom_llm_provider, litellm_metadata)
if rejection is not None:
_raise_live_session_rejection(rejection)
resolved_api_base: Final = api_base or litellm.api_base or "https://api.openai.com/"
await openai_live_sessions.async_live_session(
model=model,
websocket=websocket,
logging_obj=litellm_logging_obj,
session_start=session_start,
api_base=api_base or litellm.api_base or "https://api.openai.com/",
api_key=api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY"),
api_base=resolved_api_base,
api_key=await resolve_openai_bearer_token_async(
static_api_key=openai_static_api_key(api_key), api_base=resolved_api_base, litellm_params=litellm_params
),
)
@ -499,6 +519,7 @@ async def _arealtime(
api_base=dynamic_api_base or litellm_params.api_base,
api_key=dynamic_api_key,
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_params=kwargs,
)
return
@ -558,8 +579,9 @@ async def _arealtime(
)
elif _custom_llm_provider == "openai":
api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/"
# set API KEY
api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
api_key = await resolve_openai_bearer_token_async(
static_api_key=openai_static_api_key(dynamic_api_key), api_base=api_base, litellm_params=kwargs
)
await openai_realtime.async_realtime(
model=model,
@ -774,9 +796,18 @@ async def realtime_health_check(
str | None, api_version or resolved_params.get("api_version")
)
url: str | None = None
health_check_api_key: Final = (
await resolve_openai_bearer_token_async(
static_api_key=openai_static_api_key(resolved_api_key),
api_base=resolved_api_base,
litellm_params=resolved_params,
)
if custom_llm_provider == LlmProviders.OPENAI.value
else resolved_api_key
)
auth_headers: Final = _realtime_health_check_auth_headers(
custom_llm_provider=custom_llm_provider,
api_key=resolved_api_key,
api_key=health_check_api_key,
model_params=resolved_params,
)
if custom_llm_provider == "azure":

35
litellm/systemone/main.py
View file

@ -1,4 +1,6 @@
import asyncio
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
@ -78,21 +80,24 @@ async def asystemone(
assets: DecisionsAssetEntries | None = None,
**kwargs: object, # kwargs-ok: litellm_params forwarded by @client
) -> DecisionsResponse:
call: Final = _prepare_systemone_call(
model=model,
state=state,
questions=questions,
images=images,
assets=assets,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
call: Final = await asyncio.to_thread(
partial(
_prepare_systemone_call,
model=model,
state=state,
questions=questions,
images=images,
assets=assets,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return _systemone_response(await asend(call))

3
litellm/types/router.py
View file

@ -1407,8 +1407,9 @@ def reject_server_owned_wif_params(body: Mapping[str, object]) -> None:
This lives here rather than under ``litellm.proxy`` so the router can call it on a
post-authentication merge without core importing from the proxy package.
"""
submitted_names: Final = frozenset(key.removesuffix("[]") for key in body)
for param in _server_owned_wif_litellm_params:
if param in body:
if param in submitted_names:
raise ValueError(
f"Rejected Request: {param} is a server-owned workload identity federation or OAuth token exchange "
"parameter "

25
tests/integration/compatibility/test_audio_transcriptions_wire.py
View file

@ -601,6 +601,31 @@ def test_audio_transcription_consumes_bracketed_form_metadata_tags(gateway: Gate
) == (tag,), rows
@pytest.mark.parametrize(
"identity_param", ("openai_identity_provider_id", "openai_service_account_id", "openai_identity_token_file")
)
def test_audio_transcription_refuses_a_bracketed_identity_form_key_before_the_upstream(
gateway: Gateway, identity_param: str
) -> None:
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
def respond(request: Request) -> Reply:
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
)
response: Final = gateway.request_multipart(
"/v1/audio/transcriptions",
{"model": model, "response_format": "json", f"{identity_param}[]": "caller-chosen-" + uuid.uuid4().hex},
{"file": ("a.wav", wav, _WAV_CONTENT_TYPE)},
)
assert response.status_code == 401, response.text
assert identity_param in response.text, response.text
assert wire.drain() == ()
def test_audio_transcription_preserves_the_client_file_content_type(gateway: Gateway) -> None:
pytest.skip(
"BUG: the proxy discards the client's file part content type and re-derives it from the filename, "

67
tests/integration/routing/test_user_config_routing.py
View file

@ -3,8 +3,10 @@ import uuid
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, wire_server
USER_KEY: Final = "sk-user-supplied-" + uuid.uuid4().hex
@ -30,18 +32,13 @@ def _completion(request: Request) -> Reply:
)
def _deployment_params(upstream_url: str) -> dict[str, object]:
return {"model": "openai/gpt-4.1-mini", "api_base": upstream_url + "/v1", "api_key": USER_KEY}
def _user_config(upstream_url: str) -> dict[str, object]:
return {
"model_list": [
{
"model_name": "user-config-deployment",
"litellm_params": {
"model": "openai/gpt-4.1-mini",
"api_base": upstream_url + "/v1",
"api_key": USER_KEY,
},
}
],
"model_list": [{"model_name": "user-config-deployment", "litellm_params": _deployment_params(upstream_url)}],
"num_retries": 0,
}
@ -95,6 +92,54 @@ def test_user_config_routes_to_the_user_supplied_deployment_when_opted_in(gatewa
assert json.loads(completions[0].body)["model"] == "gpt-4.1-mini"
def _identity_in_deployment(upstream_url: str, identity_param: str) -> dict[str, object]:
return {
"model_list": [
{
"model_name": "user-config-deployment",
"litellm_params": {**_deployment_params(upstream_url), identity_param: "caller-chosen"},
}
],
"num_retries": 0,
}
def _identity_in_defaults(upstream_url: str, identity_param: str) -> dict[str, object]:
return {**_user_config(upstream_url), "default_litellm_params": {identity_param: "caller-chosen"}}
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def test_user_config_cannot_carry_a_server_owned_identity_param_even_when_opted_in(
gateway: Gateway, tmp_path: Path
) -> None:
cases: Final = (
(_identity_in_deployment, "openai_identity_provider_id"),
(_identity_in_defaults, "openai_service_account_id"),
)
with wire_server(_completion) as upstream:
config: Final = _opt_in_config(tmp_path, upstream.url)
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
responses: Final = tuple(
(
identity_param,
candidate.request(
"POST",
"/v1/chat/completions",
{
"model": "user-config-deployment",
"messages": [{"role": "user", "content": "user config identity"}],
"user_config": placement(upstream.url, identity_param),
},
),
)
for placement, identity_param in cases
)
for identity_param, response in responses:
assert response.status_code == 401, response.text
assert identity_param in response.text, response.text
assert tuple(request for request in upstream.drain() if request.target == "/v1/chat/completions") == ()
def test_user_config_is_rejected_without_the_opt_in(gateway: Gateway) -> None:
with wire_server(_completion) as upstream:
response: Final = gateway.request("POST", "/v1/chat/completions", _request_body(upstream.url))

14
tests/unit/batches/test_batch_utils.py
View file

@ -2014,6 +2014,20 @@ def test_total_cost_bills_cached_tokens_per_line_at_the_batch_cached_rate():
assert result.cost == pytest.approx(long_line + short_line)
def test_extract_credentials_keeps_openai_workload_identity_params():
params = {
"openai_identity_provider_id": "idp_deployment",
"openai_service_account_id": "user-deployment",
"openai_identity_token_file": "/var/run/secrets/token",
"model": "gpt-4o",
}
assert bu._extract_file_access_credentials(params) == {
"openai_identity_provider_id": "idp_deployment",
"openai_service_account_id": "user-deployment",
"openai_identity_token_file": "/var/run/secrets/token",
}
# --------------------------------------------------------------------------- #
# batch_cost_is_final
# --------------------------------------------------------------------------- #

2
tests/unit/llms/openai/completion/test_handler.py
View file

@ -106,7 +106,7 @@ async def test_acompletion_uses_optimized_http_client():
with patch.object(BaseOpenAILLM, "_get_async_http_client", return_value=mock_http_client) as mock_get_client:
with patch(
"litellm.llms.openai.completion.handler.AsyncOpenAI",
"litellm.llms.openai.workload_identity.AsyncOpenAI",
return_value=mock_async_openai,
) as mock_openai_class:
handler = OpenAITextCompletion()

25
tests/unit/llms/openai/test_openai_common_utils.py
View file

@ -465,15 +465,26 @@ def _sdk_api_client(
)
if api == "image_variations":
variation_factory: Final = OpenAIImageVariationsHandler()
params: Final = {
"api_key": "transport-only",
"base_url": kwargs["api_base"],
"timeout": timeout,
}
return (
variation_factory.get_async_client(client=client, init_client_params=params)
variation_factory.get_async_client(
client=client,
api_key="transport-only",
api_base="https://sdk-default.example/v1",
timeout=timeout,
max_retries=0,
organization=None,
litellm_params={},
)
if is_async
else variation_factory.get_sync_client(client=client, init_client_params=params)
else variation_factory.get_sync_client(
client=client,
api_key="transport-only",
api_base="https://sdk-default.example/v1",
timeout=timeout,
max_retries=0,
organization=None,
litellm_params={},
)
)
if api == "azure_gateway":
return BaseAzureLLM()._init_azure_client_for_cloudflare_ai_gateway(

1311
tests/unit/llms/openai/test_openai_workload_identity.py
View file

File diff suppressed because it is too large Load diff

36
tests/unit/proxy/auth/test_auth_utils.py
View file

@ -35,7 +35,7 @@ from litellm.proxy.auth.auth_utils import (
is_request_body_safe,
)
from litellm.router import Router
from litellm.types.utils import oauth_token_exchange_litellm_params
from litellm.types.utils import oauth_token_exchange_litellm_params, server_owned_wif_litellm_params
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
@ -163,13 +163,45 @@ def test_every_server_owned_identity_param_is_refused_from_a_request_body(param:
)
def _client_router_config(placement: str, litellm_params: dict[str, str]) -> dict[str, object]:
if placement == "model_list":
return {"model_list": [{"model_name": "gpt-5.4-nano", "litellm_params": litellm_params}]}
return {"default_litellm_params": litellm_params}
@pytest.mark.parametrize("param", server_owned_wif_litellm_params)
@pytest.mark.parametrize("placement", ["model_list", "default_litellm_params"])
def test_every_server_owned_auth_param_is_refused_from_a_client_router_config(param: str, placement: str):
user_config: Final = _client_router_config(placement, {"model": "openai/gpt-5.4-nano", param: "attacker-chosen"})
with pytest.raises(
ValueError,
match="server-owned workload identity federation or OAuth token exchange parameter",
):
is_request_body_safe(
request_body={"model": "gpt-5.4-nano", "user_config": user_config},
general_settings={"allow_client_side_credentials": True},
llm_router=None,
model="gpt-5.4-nano",
)
@pytest.mark.parametrize(
"body",
[
{"model": "claude-sonnet-5", "litellm_credential_name": "admin-wif"},
{"model": "claude-sonnet-5", "litellm_params": {"litellm_credential_name": "admin-wif"}},
{
"model": "claude-sonnet-5",
"user_config": _client_router_config(
"model_list", {"model": "anthropic/claude-sonnet-5", "litellm_credential_name": "admin-wif"}
),
},
{
"model": "claude-sonnet-5",
"user_config": _client_router_config("default_litellm_params", {"litellm_credential_name": "admin-wif"}),
},
],
ids=["top_level", "nested_litellm_params"],
ids=["top_level", "nested_litellm_params", "client_router_model_list", "client_router_defaults"],
)
def test_a_request_body_cannot_pick_a_federated_identity_by_credential_name(monkeypatch, body: dict):
"""Naming a federated credential moves the token exchange onto that credential's federation rule

24
tests/unit/proxy/batches_endpoints/test_endpoints.py
View file

@ -35,6 +35,7 @@ import logging
from contextlib import ExitStack
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
@ -521,6 +522,29 @@ async def test_create__fallback_explicit_provider_bypasses_not_found_gate(harnes
assert harness.acreate_kwargs()["custom_llm_provider"] == "anthropic"
@pytest.mark.asyncio
async def test_create__fallback_workload_identity_alone_forwards(
harness: Harness, no_openai_creds: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
token_file = tmp_path / "subject_token.jwt"
token_file.write_text("subject-token-from-file")
monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123")
monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456")
monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file))
set_body(
harness,
{
"input_file_id": "file-plain",
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
},
)
await call_create(harness)
assert harness.acreate_kwargs()["custom_llm_provider"] == "openai"
@pytest.mark.asyncio
async def test_create__fallback_env_key_alone_forwards(harness, monkeypatch):
monkeypatch.setattr(litellm, "api_key", None)

160
tests/unit/proxy/guardrails/guardrail_hooks/openai/test_moderations.py
View file

@ -5,6 +5,7 @@ Test OpenAI Moderation Guardrail
import json
import os
from pathlib import Path
from typing import Final
@ -12,13 +13,22 @@ from unittest.mock import MagicMock, patch
import httpx
import pytest
import respx
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.openai.workload_identity import _workload_identity_auth
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import get_logging_caching_headers
from litellm.proxy.guardrails.guardrail_hooks.openai import (
initialize_guardrail as initialize_openai_moderation_guardrail,
)
from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import (
OpenAIModerationGuardrail,
)
from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER, InMemoryGuardrailHandler
from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2
from litellm.types.guardrails import Guardrail, LitellmParams, SupportedGuardrailIntegrations
from litellm.types.llms.openai import OpenAIModerationResponse, OpenAIModerationResult
@ -1089,3 +1099,153 @@ async def test_openai_moderation_responses_api_input_field():
input_type="request",
)
assert "Violated OpenAI moderation policy" in str(chat_error.value)
class _ModerationsEndpointThatRecordsTheAuthorization:
authorization: str | None = None
def __call__(self, request: httpx.Request) -> httpx.Response:
self.authorization = request.headers.get("Authorization")
return httpx.Response(
200,
json={
"id": "modr-federated",
"model": "omni-moderation-latest",
"results": [
{"flagged": False, "categories": {}, "category_scores": {}, "category_applied_input_types": {}}
],
},
)
@pytest.mark.asyncio
@respx.mock
async def test_openai_moderation_sends_the_federated_bearer_when_only_workload_identity_is_set(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
token_file: Final = tmp_path / "subject_token.jwt"
token_file.write_text("subject-token")
for name in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE"):
monkeypatch.delenv(name, raising=False)
for attribute in ("api_key", "openai_key", "api_base"):
monkeypatch.setattr(litellm, attribute, None)
monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_guardrail")
monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-guardrail")
monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file))
_workload_identity_auth.cache_clear()
respx.post("https://auth.openai.com/oauth/token").mock(
return_value=httpx.Response(200, json={"access_token": "guardrail-bearer", "expires_in": 3600})
)
endpoint: Final = _ModerationsEndpointThatRecordsTheAuthorization()
http_client: Final = AsyncHTTPHandler()
http_client.client = httpx.AsyncClient(transport=httpx.MockTransport(endpoint))
guardrail: Final = OpenAIModerationGuardrail(guardrail_name="openai-mod")
guardrail.async_handler = http_client
response: Final = await guardrail.async_make_request("hello")
assert response.id == "modr-federated"
assert endpoint.authorization == "Bearer guardrail-bearer"
@pytest.mark.asyncio
@respx.mock
async def test_openai_moderation_federates_with_the_identity_in_its_own_guardrail_config(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
token_file: Final = tmp_path / "subject_token.jwt"
token_file.write_text("subject-token")
for name in (
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"OPENAI_API_BASE",
"OPENAI_IDENTITY_PROVIDER_ID",
"OPENAI_SERVICE_ACCOUNT_ID",
"OPENAI_IDENTITY_TOKEN_FILE",
):
monkeypatch.delenv(name, raising=False)
for attribute in ("api_key", "openai_key", "api_base"):
monkeypatch.setattr(litellm, attribute, None)
monkeypatch.setenv("GUARDRAIL_IDENTITY_PROVIDER_ID", "idp_guardrail_config")
_workload_identity_auth.cache_clear()
exchange: Final = respx.post("https://auth.openai.com/oauth/token").mock(
return_value=httpx.Response(200, json={"access_token": "guardrail-config-bearer", "expires_in": 3600})
)
endpoint: Final = _ModerationsEndpointThatRecordsTheAuthorization()
http_client: Final = AsyncHTTPHandler()
http_client.client = httpx.AsyncClient(transport=httpx.MockTransport(endpoint))
guardrail_config: Final = LitellmParams(
guardrail=SupportedGuardrailIntegrations.OPENAI_MODERATION,
mode="pre_call",
openai_identity_provider_id="os.environ/GUARDRAIL_IDENTITY_PROVIDER_ID",
openai_service_account_id="user-guardrail-config",
openai_identity_token_file=str(token_file),
)
monkeypatch.setattr(litellm, "callbacks", [])
guardrail: Final = initialize_openai_moderation_guardrail(
litellm_params=guardrail_config,
guardrail=Guardrail(guardrail_name="openai-mod", litellm_params=guardrail_config),
)
guardrail.async_handler = http_client
await guardrail.async_make_request("hello")
exchange_body: Final = json.loads(exchange.calls.last.request.content)
assert exchange_body["identity_provider_id"] == "idp_guardrail_config"
assert exchange_body["service_account_id"] == "user-guardrail-config"
assert endpoint.authorization == "Bearer guardrail-config-bearer"
@pytest.fixture
def no_ambient_openai_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
for name in (
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"OPENAI_API_BASE",
"OPENAI_IDENTITY_PROVIDER_ID",
"OPENAI_SERVICE_ACCOUNT_ID",
"OPENAI_IDENTITY_TOKEN_FILE",
):
monkeypatch.delenv(name, raising=False)
for attribute in ("api_key", "openai_key", "api_base"):
monkeypatch.setattr(litellm, attribute, None)
monkeypatch.setattr(litellm, "callbacks", [])
PARTIAL_IDENTITY_MODERATION: Final = {
"guardrail": SupportedGuardrailIntegrations.OPENAI_MODERATION.value,
"mode": "pre_call",
"openai_identity_provider_id": "idp_without_the_rest",
}
@pytest.mark.usefixtures("no_ambient_openai_credentials")
def test_openai_moderation_with_a_partial_identity_is_a_configuration_error_naming_the_gap() -> None:
with pytest.raises(ValueError, match="openai_service_account_id, openai_identity_token_file"):
InMemoryGuardrailHandler().initialize_guardrail(
guardrail={"guardrail_name": "partial-identity-mod", "litellm_params": PARTIAL_IDENTITY_MODERATION},
source="db",
)
@pytest.mark.usefixtures("no_ambient_openai_credentials")
def test_proxy_boot_skips_an_openai_moderation_with_a_partial_identity() -> None:
IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.clear()
IN_MEMORY_GUARDRAIL_HANDLER.guardrail_id_to_custom_guardrail.clear()
init_guardrails_v2(
all_guardrails=[
{"guardrail_name": "partial-identity-mod", "litellm_params": PARTIAL_IDENTITY_MODERATION},
{
"guardrail_name": "keyed-mod",
"litellm_params": {
"guardrail": SupportedGuardrailIntegrations.OPENAI_MODERATION.value,
"mode": "pre_call",
"api_key": "sk-moderation",
},
},
]
)
booted: Final = IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.values()
assert {guardrail["guardrail_name"] for guardrail in booted} == {"keyed-mod"}

124
tests/unit/realtime_api/test_main.py
View file

@ -2,17 +2,20 @@ import asyncio
import json
import time
from collections.abc import Awaitable, Callable, Mapping
from pathlib import Path
from types import TracebackType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
import litellm
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.openai.workload_identity import _workload_identity_auth
from litellm.models.credentials import CredentialItem
from litellm.realtime_api import main as realtime_main
from litellm.realtime_api.main import _with_resolved_session_model
@ -1052,3 +1055,124 @@ async def test_voice_transcript_blocked_by_guardrail():
finally:
litellm.callbacks = []
@pytest.fixture
def federated_deployment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> dict[str, str]:
token_file: Final = tmp_path / "subject_token.jwt"
token_file.write_text("subject-token")
for name in (
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"OPENAI_API_BASE",
"OPENAI_IDENTITY_PROVIDER_ID",
"OPENAI_SERVICE_ACCOUNT_ID",
"OPENAI_IDENTITY_TOKEN_FILE",
):
monkeypatch.delenv(name, raising=False)
for attribute in ("api_key", "openai_key", "api_base"):
monkeypatch.setattr(litellm, attribute, None)
_workload_identity_auth.cache_clear()
return {
"openai_identity_provider_id": "idp_realtime",
"openai_service_account_id": "user-realtime",
"openai_identity_token_file": str(token_file),
}
def _mock_token_exchange() -> None:
respx.post("https://auth.openai.com/oauth/token").mock(
return_value=httpx.Response(200, json={"access_token": "federated-bearer", "expires_in": 3600})
)
class _ConnectThatStopsAfterCapturingTheAuthorization:
authorization: str | None = None
def __call__(
self, url: str, *, additional_headers: Mapping[str, str], **kwargs: object
) -> "_ConnectThatStopsAfterCapturingTheAuthorization":
self.authorization = additional_headers.get("Authorization")
return self
async def __aenter__(self) -> None:
raise RuntimeError("backend headers captured, nothing to bridge")
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
return None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"live_session_start",
[None, {"type": "session.start", "session": {"model": "gpt-live-1"}}],
ids=["realtime", "live_session"],
)
@respx.mock
async def test_openai_websocket_sessions_send_the_federated_bearer_for_a_keyless_deployment(
federated_deployment: dict[str, str], live_session_start: Mapping[str, object] | None
) -> None:
_mock_token_exchange()
connect: Final = _ConnectThatStopsAfterCapturingTheAuthorization()
with patch("websockets.connect", connect):
await realtime_main._arealtime.__wrapped__(
model="gpt-realtime",
custom_llm_provider="openai",
websocket=MagicMock(close=AsyncMock()),
litellm_logging_obj=FakeLogging(),
live_session_start=live_session_start,
**federated_deployment,
)
assert connect.authorization == "Bearer federated-bearer"
class _GatewayThatRecordsTheAuthorization:
authorization: str | None = None
def __call__(self, request: httpx.Request) -> httpx.Response:
self.authorization = request.headers.get("Authorization")
return httpx.Response(200, json={"value": "ek_test"})
@pytest.mark.asyncio
@pytest.mark.parametrize(
"route",
[realtime_main.acreate_realtime_client_secret, realtime_main.acreate_realtime_transcription_session],
ids=["client_secrets", "transcription_sessions"],
)
@respx.mock
async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyless_deployment(
federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]]
) -> None:
_mock_token_exchange()
gateway: Final = _GatewayThatRecordsTheAuthorization()
await route.__wrapped__(
model="gpt-realtime",
custom_llm_provider="openai",
litellm_logging_obj=FakeLogging(),
client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)),
**federated_deployment,
)
assert gateway.authorization == "Bearer federated-bearer"
@pytest.mark.asyncio
@respx.mock
async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_keyless_deployment(
federated_deployment: dict[str, str],
) -> None:
_mock_token_exchange()
connect: Final = _CapturingConnect()
with patch("websockets.connect", connect):
assert await realtime_main.realtime_health_check(
model="gpt-realtime",
custom_llm_provider="openai",
api_key=None,
model_params={"model": "openai/gpt-realtime", **federated_deployment},
)
assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer federated-bearer"}

8
tests/unit/test_private_usage_aliases.py
View file

@ -4890,6 +4890,14 @@ LLMS_FORWARDER_CASES: Final = (
"staticmethod",
False,
),
(
"litellm.llms.openai.common_utils",
"BaseOpenAILLM",
"_get_sync_http_client",
"get_sync_http_client",
"staticmethod",
False,
),
(
"litellm.llms.openai.openai",
"OpenAIChatCompletion",

6
tests/unit/types/test_router.py
View file

@ -372,3 +372,9 @@ def test_server_owned_wif_fields_present_reports_openai_fields():
def test_reject_server_owned_wif_params_names_each_openai_field(param_name: str):
with pytest.raises(ValueError, match=param_name):
reject_server_owned_wif_params({param_name: "client-supplied"})
@pytest.mark.parametrize("param_name", openai_wif_litellm_params)
def test_reject_server_owned_wif_params_catches_the_array_form_key(param_name: str):
with pytest.raises(ValueError, match=param_name):
reject_server_owned_wif_params({f"{param_name}[]": "client-supplied"})