Merge remote-tracking branch 'origin/main' into litellm_ws_spend_log_per_turn

# Conflicts:
#	litellm/proxy/response_api_endpoints/endpoints.py

.github/scripts/assert_ci_coverage.py

@@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
 DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
 COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
 GLOB_CHARS = frozenset("*?")
+DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
 
 # Trees whose jobs are sharded with no catch-all bucket, so every child that holds
 # tests has to be named by some shard or it runs nowhere. A child listed here is
@@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
     return frozenset(ast.literal_eval(values))
 
 
+def _owns(workflow: str, relative_path: str) -> bool:
+    owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
+    return owned_root is not None and relative_path.startswith(owned_root)
+
+
+def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
+    return frozenset[str]().union(
+        *(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
+    )
+
+
 def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
     runner: Final = repo_root / "tests/integration/run.py"
     if not runner.exists():
@@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         )
         != 1
     )
-    gha_tokens: Final = _invoked_test_tokens(
-        scalar
-        for path in (repo_root / ".github/workflows").glob("*.y*ml")
-        for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
+    workflow_tokens: Final = MappingProxyType(
+        {
+            path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
+            for path in (repo_root / ".github/workflows").glob("*.y*ml")
+        }
+    )
+    gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
+    circle_owned: Final = (
+        frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
     )
     findings: Final = (
         tuple(
-            Finding(path, "integration contract is also selected by GitHub Actions")
-            for path in paths
+            Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
+            for path in sorted(circle_owned)
             if any(_token_covers(token, path) for token in gha_tokens)
         )
         + tuple(
@@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
             for path in sorted(github_files)
             if not (repo_root / path).is_file()
         )
+        + tuple(
+            Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
+            for path in sorted(github_files)
+            if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
+        )
     )
     browser_commands: Final = tuple(
         scalar.value
@@ -688,13 +710,9 @@ def main() -> int:
     scalars = _all_scalars()
 
     integration_paths, ownership_findings = _integration_ownership()
-    test_findings = (
-        _uncovered_tests(
-            allowlist,
-            _invoked_selections(scalars)
-            + (Selection(included=integration_paths, ignored=frozenset()),),
-        )
-        + ownership_findings
+    test_findings = _uncovered_tests(
+        allowlist,
+        _invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
     )
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
     stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@@ -711,13 +729,25 @@ def main() -> int:
             test_findings,
             "Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
         )
+    if ownership_findings:
+        browser_remedy = (
+            " CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
+            "select them and the default Playwright config must exclude them."
+            if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
+            else ""
+        )
+        _report(
+            "integration contracts with the wrong CI owner",
+            ownership_findings,
+            "Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
+        )
     if dockerfile_findings:
         _report(
             "Dockerfiles that no CI job builds",
             dockerfile_findings,
             "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
         )
-    if stale_findings or test_findings or dockerfile_findings:
+    if stale_findings or test_findings or ownership_findings or dockerfile_findings:
         return 1
 
     _write(

AGENTS.md

@@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
 
 `tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
-End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
+End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
 
 When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
@@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
 
 Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
 
-If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
+Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
 
 Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
 

litellm/__init__.py

@@ -1177,7 +1177,9 @@ def _build_models_by_provider() -> dict:
         "edenai": edenai_models,
         "datarobot": datarobot_models,
         "vertex_ai": vertex_chat_models
+        | vertex_code_chat_models
         | vertex_text_models
+        | vertex_code_text_models
         | vertex_anthropic_models
         | vertex_vision_models
         | vertex_language_models
@@ -1185,7 +1187,7 @@ def _build_models_by_provider() -> dict:
         | vertex_minimax_models
         | vertex_moonshot_models
         | vertex_zai_models,
-        "ai21": ai21_models,
+        "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
         "ollama": ollama_models,
@@ -1416,6 +1418,7 @@ from .exceptions import (
     InvalidRequestError,
     BadRequestError,
     ImageFetchError,
+    MalformedRequestFieldError as MalformedRequestFieldError,
     VectorStoreSearchError,
     NotFoundError,
     PermissionDeniedError,

litellm/assistants/main.py

@@ -129,6 +129,7 @@ def get_assistants(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_assistants=aget_assistants,
         )
@@ -312,6 +313,7 @@ def create_assistants(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             create_assistant_data=create_assistant_data,
             client=client,
             async_create_assistants=async_create_assistants,
@@ -461,6 +463,7 @@ def delete_assistant(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             assistant_id=assistant_id,
             client=client,
             async_delete_assistants=async_delete_assistants,
@@ -637,6 +640,7 @@ def create_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             acreate_thread=acreate_thread,
         )
@@ -783,6 +787,7 @@ def get_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_thread=aget_thread,
         )
@@ -960,6 +965,7 @@ def add_message(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             a_add_message=a_add_message,
         )
@@ -1111,6 +1117,7 @@ def get_messages(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_messages=aget_messages,
         )
@@ -1308,6 +1315,7 @@ def run_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             arun_thread=arun_thread,
             event_handler=event_handler,

litellm/batches/batch_utils.py

@@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
 )
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import ModelInfo, Usage
-from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
+from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
 from litellm.utils import token_counter
 
 
@@ -569,6 +569,7 @@ def extract_file_access_credentials(
             # A federated deployment holds no api_key, so without these the fetch that reads a
             # finished batch's output has nothing to authenticate with and its cost is never billed.
             *sorted(ANTHROPIC_WIF_KWARGS_KEYS),
+            *sorted(OPENAI_WIF_KWARGS_KEYS),
         )
         for key in credential_keys:
             if key in litellm_params:

litellm/batches/main.py

@@ -309,6 +309,7 @@ def create_batch(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
             organization=organization,
             timeout=timeout,
             max_retries=optional_params.max_retries,
+            litellm_params=litellm_params,
         )
     elif custom_llm_provider == "azure":
         api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -819,6 +821,7 @@ def list_batches(
                 organization=organization,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -1016,6 +1019,7 @@ def cancel_batch(
                 organization=organization,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
     return not _supports_audio_input(model, litellm_params)
 
 
-def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
+def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
     return MappingProxyType(
         {
             key: value
@@ -1455,7 +1455,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
-        for key, value in _provider_metadata(raw_response.model_extra).items():
+        for key, value in responses_provider_metadata(raw_response.model_extra).items():
             setattr(model_response, key, value)
 
         # Preserve hidden params from the ResponsesAPIResponse, especially the headers
@@ -2139,7 +2139,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 from litellm.responses.utils import ResponseAPILoggingUtils
 
                 usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
-            provider_metadata: Final = _provider_metadata(response_data)
+            provider_metadata: Final = responses_provider_metadata(response_data)
             served_service_tier: Final = response_data.get("service_tier")
             return ModelResponseStream(
                 choices=[

litellm/constants.py

@@ -1859,6 +1859,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
 PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
 PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
 PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
+TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)  # 1 minute
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
 RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))

litellm/containers/README.md

@@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
         self,
         headers: dict,
         api_key: Optional[str] = None,
+        litellm_params: Optional[GenericLiteLLMParams] = None,
     ) -> dict:
         """Azure uses api-key header instead of Bearer token."""
         import litellm

litellm/cost_calculator.py

@@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
 from litellm.llms.openai.cost_calculation import (
     video_output_cost_per_second,
 )
+from litellm.llms.openai.transcriptions.gpt_transformation import (
+    OpenAIGPTAudioTranscriptionConfig,
+)
 from litellm.llms.perplexity.cost_calculator import (
     cost_per_token as perplexity_cost_per_token,
 )
@@ -325,6 +328,15 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
+def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
+    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+        return False
+    transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
+        model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
+    )
+    return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
+
+
 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
 
 
@@ -1739,6 +1751,12 @@ def completion_cost(
                     audio_transcription_file_duration = _hidden.get(
                         "audio_transcription_duration",
                         getattr(completion_response, "duration", 0.0),
+                    ) or (
+                        completion_response.usage.seconds
+                        if isinstance(completion_response, TranscriptionResponse)
+                        and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
+                        and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
+                        else 0.0
                     )
                 elif call_type in _RERANK_CALL_TYPES:
                     if completion_response is not None and isinstance(completion_response, RerankResponse):

litellm/decisions/call.py

@@ -458,7 +458,7 @@ def prepare_call(
         ) from error
 
     try:
-        resolved_api_base, resolved_api_key = provider_config.resolve_connection(
+        resolved_api_base, static_api_key = provider_config.resolve_connection(
             dynamic_api_base or api_base, dynamic_api_key or api_key
         )
     except ValueError as error:
@@ -469,6 +469,7 @@ def prepare_call(
             model=model,
             llm_provider=provider,
         )
+    resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
     if resolved_api_key is None and provider_config.api_key_required:
         raise litellm.AuthenticationError(
             message=f"Missing API key for Decisions provider '{provider}'",

litellm/decisions/main.py

@@ -1,5 +1,7 @@
+import asyncio
 import warnings
 from collections.abc import Mapping
+from functools import partial
 from typing import Final
 
 import httpx
@@ -104,19 +106,22 @@ async def adecisions(
     videos: DecisionsVideoClips | None = None,
     **kwargs: object,
 ) -> OpenAIDecisionResponse | DecisionsResponse:
-    call: Final = _prepare_decisions_call(
-        model=model,
-        decision_input=input,
-        questions=questions,
-        safety_identifier=safety_identifier,
-        audio=audio,
-        videos=videos,
-        api_key=api_key,
-        api_base=api_base,
-        timeout=timeout,
-        custom_llm_provider=custom_llm_provider,
-        extra_headers=extra_headers,
-        kwargs=kwargs,
+    call: Final = await asyncio.to_thread(
+        partial(
+            _prepare_decisions_call,
+            model=model,
+            decision_input=input,
+            questions=questions,
+            safety_identifier=safety_identifier,
+            audio=audio,
+            videos=videos,
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            custom_llm_provider=custom_llm_provider,
+            extra_headers=extra_headers,
+            kwargs=kwargs,
+        )
     )
     return await asend(call)
 

litellm/exceptions.py

@@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
         )
 
 
+class MalformedRequestFieldError(BadRequestError):
+    pass
+
+
 class UnprocessableEntityError(openai.UnprocessableEntityError):
     def __init__(
         self,

litellm/files/main.py

@@ -259,6 +259,7 @@ def create_file(
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
                 create_file_data=_create_file_request,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -388,6 +389,7 @@ def file_retrieve(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -569,6 +571,7 @@ def file_delete(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -775,6 +778,7 @@ def file_list(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -972,6 +976,7 @@ def file_content(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -1093,6 +1098,7 @@ def file_content_streaming(
             organization=openai_creds.organization,
             chunk_size=chunk_size,
             client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
+            litellm_params=optional_params.model_dump(exclude_none=True),
         )
     elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
         if not _is_async:

litellm/fine_tuning/main.py

@@ -239,6 +239,7 @@ def create_fine_tuning_job(
                 client=kwargs.get(
                     "client", None
                 ),  # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":

litellm/images/main.py

@@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes
 # BFL handlers
 from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
 from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
+from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
 from litellm.main import (
     azure_chat_completions,
     base_llm_aiohttp_handler,
@@ -505,6 +506,7 @@ def image_generation(
                 aimg_generation=aimg_generation,
                 client=client,
                 headers=headers,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "bedrock":
             if model is None:
@@ -681,10 +683,13 @@ def image_variation(
     api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
 
     if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
-        if api_key is None:
-            raise ValueError("API key is required for OpenAI image variations")
         if api_base is None:
             raise ValueError("API base is required for OpenAI image variations")
+        if api_key is None and (
+            resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
+            is None
+        ):
+            raise ValueError("API key is required for OpenAI image variations")
 
         response = openai_image_variations.image_variations(
             model_response=model_response,

litellm/integrations/prometheus.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     PROXY_REJECTED_BEFORE_ROUTING_KEY,
 )
 from litellm.exceptions import (
+    MalformedRequestFieldError,
     validate_rate_limit_category,
     validate_rate_limit_type,
 )
@@ -3104,7 +3105,11 @@ class PrometheusLogger(CustomLogger):
 
             # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
             # into requested_model and leave deployment-scoped labels empty.
-            deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
+            deployment_selected: Final = (
+                bool(model_id)
+                and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
+                and not isinstance(exception, MalformedRequestFieldError)
+            )
             if deployment_selected:
                 label_litellm_model_name = litellm_model_name
                 label_model_id = model_id

litellm/litellm_core_utils/get_litellm_params.py

@@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "github_copilot_auth_type",
             "github_copilot_user_session",
             "fireworks_forward_user_id",
+            "responses_api_version",
             PROVIDER_AFFINITY_HEADER_KWARG_KEY,
         }
     )

litellm/litellm_core_utils/get_model_cost_map.py

@@ -17,6 +17,7 @@ import random
 import sys
 import threading
 import time
+import urllib.parse
 from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass, replace
 from datetime import datetime, timezone
@@ -24,11 +25,11 @@ from importlib.resources import files
 from itertools import chain
 from pathlib import Path
 from types import MappingProxyType
-from typing import Final, Protocol
+from typing import Final, Protocol, cast  # noqa: TID251  # adapts boto3 and botocore's dynamic client responses
 
 import httpx
 from pydantic import ConfigDict, TypeAdapter
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm import verbose_logger
 from litellm.constants import (
@@ -266,6 +267,19 @@ class _SyncGetClient(Protocol):
     def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
 
 
+class _S3Body(Protocol):
+    def read(self) -> bytes: ...
+
+
+class _S3GetObjectResponse(TypedDict):
+    Body: ReadOnly[_S3Body]
+    ETag: NotRequired[ReadOnly[str]]
+
+
+class _S3Client(Protocol):
+    def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
+
+
 _FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
 
 
@@ -276,6 +290,20 @@ def _default_reload_client() -> _AsyncGetClient:
     return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
 
 
+def _default_s3_client(timeout: int) -> _S3Client:
+    import boto3
+    from botocore.config import Config
+
+    return cast(  # cast-ok: boto3's generic client factory does not expose its generated S3 response type
+        _S3Client,
+        boto3.client(  # pyright: ignore[reportUnknownMemberType]  # generic factory returns Unknown service clients
+            "s3",
+            region_name=os.getenv("AWS_REGION_NAME") or None,
+            config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
+        ),
+    )
+
+
 def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
     reason: Final = f"{type(error).__name__} fetching {url}: {error}"
     if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
@@ -283,7 +311,41 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
     return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
 
 
+def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
+    try:
+        parsed: Final[object] = json.loads(content)
+    except ValueError as e:
+        return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
+    if not isinstance(parsed, dict):
+        return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
+    model_cost_map: Final = cast(  # cast-ok: JSON object keys are strings by definition
+        dict[str, object], parsed
+    )
+    return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
+
+
+def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
+    from botocore.exceptions import BotoCoreError, ClientError
+
+    parts: Final = urllib.parse.urlsplit(url)
+    bucket: Final = parts.netloc
+    key: Final = parts.path.lstrip("/")
+    if not bucket or not key:
+        return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
+
+    try:
+        client: Final = _default_s3_client(timeout)
+        response: Final = client.get_object(Bucket=bucket, Key=key)
+        content: Final = response["Body"].read()
+    except (BotoCoreError, ClientError) as error:
+        return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
+    return _classify_fetch_content(content, url, response.get("ETag"))
+
+
 async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
+    parts: Final = urllib.parse.urlsplit(url)
+    if parts.scheme == "s3":
+        return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
     try:
         response: Final = await client.get(url, timeout=timeout)
     except (httpx.HTTPError, httpx.InvalidURL) as e:
@@ -292,6 +354,9 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
 
 
 def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
+    parts: Final = urllib.parse.urlsplit(url)
+    if parts.scheme == "s3":
+        return _attempt_fetch_s3(url, timeout)
     try:
         response: Final = client.get(url, timeout=timeout)
     except (httpx.HTTPError, httpx.InvalidURL) as e:
@@ -307,15 +372,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
         )
     if response.is_error:
         return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
-    try:
-        parsed: Final = response.json()
-    except ValueError as e:
-        return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
-    if not isinstance(parsed, dict):
-        return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
-    return ModelCostMapReloaded(
-        model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
-    )
+    return _classify_fetch_content(response.content, url, response.headers.get("etag"))
 
 
 def _next_retry_wait(

litellm/litellm_core_utils/litellm_logging.py

@@ -2311,6 +2311,13 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
 
+    def attempt_scoped_copy(self) -> "Logging":
+        """A copy with its own model_call_details, so logging one attempt of a request through it leaves
+        the state the request's other attempts share untouched."""
+        attempt: Final = copy.copy(self)
+        attempt.model_call_details = dict(self.model_call_details)
+        return attempt
+
     def carry_failed_attempt_usage(self) -> None:
         """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
         own usage while the request's failure log still bills every attempt."""
@@ -4357,6 +4364,9 @@ class Logging(LiteLLMLoggingBaseClass):
         row is built from this response rather than re-parsed from the served body."""
         self.bridged_model_response = response
 
+    def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
+        self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
+
     def billable_result(self, result: object) -> object:
         """The response the cost calculator prices: a /v1/messages body served through the chat
         bridge is replaced by the ModelResponse its spend row is built from, anything else is
@@ -4457,6 +4467,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """
         from litellm.completion_extras.litellm_responses_transformation.transformation import (
             LiteLLMResponsesTransformationHandler,
+            responses_provider_metadata,
         )
 
         served_id: Final = _provider_response_id(result)
@@ -4481,7 +4492,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 "usage-only ModelResponse to keep the spend_logs row.",
                 str(e),
             )
-            model_response: Final = litellm.ModelResponse(id=served_id)
+            model_response: Final = litellm.ModelResponse(
+                id=served_id, **responses_provider_metadata(result.model_extra)
+            )
             model_response.model = self.model
             usage: Final = result.usage
             if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage):

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking:
 
     @staticmethod
     def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
-        if "web_search_options" in kwargs:
-            web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
-            return WebSearchOptions(**web_search_options)
+        web_search_options: Final = kwargs.get("web_search_options")
+        if isinstance(web_search_options, dict):
+            return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
 
         tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
             kwargs=kwargs, tool_type="web_search_preview"
@@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking:
     _get_web_search_options = get_web_search_options
 
     @staticmethod
-    def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
-        if "tools" in kwargs:
-            return cast(list[object], kwargs.get("tools", []))
-        return None
+    def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
+        tools: Final = kwargs.get("tools")
+        return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
 
     @staticmethod
     def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

litellm/litellm_core_utils/tool_fields.py

@@ -0,0 +1,164 @@
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from functools import reduce
+from typing import Annotated, Final, TypeAlias
+
+from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
+from typing_extensions import ReadOnly, TypedDict
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+
+ROUTES_WITH_TOOLS: Final = frozenset(
+    {
+        "acompletion",
+        "aresponses",
+        "acompact_responses",
+        "anthropic_messages",
+        "agenerate_content",
+        "agenerate_content_stream",
+        "acreate_interaction",
+    }
+)
+TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
+_TOOLS: Final = TypeAdapter(list[object])
+_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
+_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
+
+
+def _schema_as_read(value: object) -> object:
+    if not value:
+        return None
+    if not isinstance(value, str):
+        return value
+    try:
+        return _JSON_OBJECT.validate_python(value)
+    except ValidationError:
+        return value
+
+
+_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
+
+
+class _ToolFunction(TypedDict, total=False):
+    parameters: ReadOnly[_Schema]
+
+
+class _ToolEntry(TypedDict, total=False):
+    function: ReadOnly[_ToolFunction | None]
+    parameters: ReadOnly[_Schema]
+    input_schema: ReadOnly[_Schema]
+    vector_store_ids: ReadOnly[list[str] | None]
+
+
+_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
+_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
+
+
+@dataclass(frozen=True, slots=True)
+class DropPolicy:
+    drop_params: bool
+    additional_drop_params: frozenset[str]
+
+    def drops(self, field: str) -> bool:
+        return self.drop_params or field in self.additional_drop_params
+
+
+@dataclass(frozen=True, slots=True)
+class MalformedField:
+    field: str
+    message: str
+    dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
+
+
+def _field_names(value: object) -> frozenset[str]:
+    try:
+        return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
+    except ValidationError:
+        return frozenset()
+
+
+def drop_policy(params: Mapping[str, object]) -> DropPolicy:
+    return DropPolicy(
+        drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
+        additional_drop_params=_field_names(params.get("additional_drop_params")),
+    )
+
+
+def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
+    error: Final = e.errors(include_url=False)[0]
+    return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
+
+
+def _tool_entry_error(index: int, entry: object) -> str | None:
+    try:
+        _TOOL_ENTRY.validate_python(entry, strict=True)
+    except ValidationError as e:
+        return first_validation_error(e, prefix=("tools", index))
+    return None
+
+
+def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
+    return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
+
+
+def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
+    tools: Final = data.get("tools")
+    if tools is None:
+        return None
+    try:
+        entries: Final = _TOOLS.validate_python(tools)
+    except ValidationError as e:
+        return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
+    entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
+    first_error: Final = next((error for error in entry_errors if error is not None), None)
+    if first_error is None:
+        return None
+    kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
+    return MalformedField(
+        "tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
+    )
+
+
+def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
+    return {name: value for name, value in data.items() if name != "web_search_options"}
+
+
+def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
+    try:
+        _WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
+    except ValidationError as e:
+        return MalformedField(
+            "web_search_options",
+            first_validation_error(e, prefix=("web_search_options",)),
+            _without_web_search_options,
+        )
+    return None
+
+
+def resolve_tool_fields(
+    data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
+) -> Mapping[str, object] | MalformedField:
+    checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
+    malformed: Final = tuple(field for field in checked if field is not None)
+    if not malformed:
+        return data
+    policies: Final = candidates()
+    rejected: Final = next(
+        (field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
+    )
+    if rejected is not None:
+        return rejected
+    return reduce(
+        lambda resolved, field: field.dropped(resolved),
+        (field for field in malformed if all(policy.drops(field.field) for policy in policies)),
+        data,
+    )
+
+
+def resolve_deployment_tool_fields(
+    kwargs: Mapping[str, object], route: object
+) -> Mapping[str, object] | MalformedField:
+    if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
+        return kwargs
+    return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
     effective_skip_tool_message_for_guardrail,
+    list_field,
     merge_guardrailed_scoped_messages,
     merge_returned_tools_into_request_tools,
     scoped_structured_message_indices,
@@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
             images=(),
         )
 
-    def extract_request_tool_names(self, data: dict) -> list[str]:
+    def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
         """Extract every tool name in an Anthropic messages request: tools[].name, plus
         tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
-        return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
+        return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
 
     @classmethod
     def _extract_input_text_and_images(

litellm/llms/anthropic/pass_through/messages/streaming_iterator.py

@@ -2,6 +2,7 @@ import asyncio
 import json
 import re
 from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
+from contextvars import ContextVar
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
 
@@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
         await stream.aclose()
 
 
+_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
+
+
+async def aclose_superseded_attempt(stream: object) -> None:
+    """Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
+    token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
+    try:
+        await aclose_if_supported(stream)
+    finally:
+        _CLOSING_SUPERSEDED_ATTEMPT.reset(token)
+
+
+def closing_superseded_attempt() -> bool:
+    return _CLOSING_SUPERSEDED_ATTEMPT.get()
+
+
 _RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
 
 

litellm/llms/anthropic/pass_through/responses_adapters/handler.py

@@ -43,6 +43,12 @@ _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
 _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
 
 
+def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
+    logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
+    if logging_obj is not None:
+        logging_obj.record_bridged_responses_api_response(result)
+
+
 def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
     """The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
     return extra_kwargs or {}
@@ -301,7 +307,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
         if not isinstance(result, ResponsesAPIResponse):
             raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
 
-        return await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
+        served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
+        _record_bridged_response(responses_kwargs, result)
+        return served
 
     @staticmethod
     def anthropic_messages_handler(

litellm/llms/azure/chat/gpt_5_transformation.py

@@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
                     status_code=400,
                     message=(
                         "Azure OpenAI does not support reasoning_effort='none' for this model. "
-                        "Supported values are: 'low', 'medium', and 'high'. "
                         "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
                         "`litellm_settings:\n drop_params: true`\n"
                         "Issue: https://github.com/BerriAI/litellm/issues/16704"

litellm/llms/azure/common_utils.py

@@ -2,6 +2,7 @@ import asyncio
 import hashlib
 import json
 import os
+import re
 from collections.abc import Callable, Mapping
 from functools import lru_cache
 from types import MappingProxyType
@@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import add_path_to_api_base
 
 azure_ad_cache: Final = DualCache()
+_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)")
 
 
 class _AzureAdTokenJson(TypedDict, total=False):
@@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM):
     ) -> dict[str, str]:  # mutable-ok: mirrors override contract
         return cls._base_validate_azure_environment(headers, litellm_params)
 
+    @staticmethod
+    def resolve_azure_api_base(api_base: str | None) -> str | None:
+        return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
+
     @staticmethod
     def _get_base_azure_url(
         api_base: str | None,
@@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM):
             default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
         """
 
-        api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
+        api_base = BaseAzureLLM.resolve_azure_api_base(api_base)
         if api_base is None:
             raise ValueError(
                 f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
@@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM):
             # ensure the request go to /openai/v1 and not just /openai
             if "/openai/v1" not in new_url:
                 parsed_url: Final = httpx.URL(new_url)
-                new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
+                new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1)))
 
         # Use the new query_params dictionary
         final_url: Final = httpx.URL(new_url).copy_with(params=query_params)

litellm/llms/azure/containers/transformation.py

@@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
         self,
         headers: dict,
         api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
         return BaseAzureLLM.base_validate_azure_environment(
             headers=headers,

litellm/llms/azure/fine_tuning/handler.py

@@ -1,4 +1,4 @@
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import Final, cast
 
 import httpx
@@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         self._ensure_training_type(create_fine_tuning_job_data)
 
@@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
         _is_async: bool = False,
         api_version: str | None = None,
-        litellm_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
         # Override to use Azure-specific client initialization
         if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI):
             client = None
 
         return self.get_azure_openai_client(
-            litellm_params=litellm_params or {},
+            litellm_params=dict(litellm_params) if litellm_params is not None else None,
             api_key=api_key,
             api_base=api_base,
             api_version=api_version,

litellm/llms/azure/responses/transformation.py

@@ -1,4 +1,7 @@
+import re
+from collections.abc import Mapping
 from copy import deepcopy
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
@@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem
 from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
+from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
@@ -25,6 +29,34 @@ else:
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
+_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+
+def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
+    value: Final = litellm_params.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None:
+    """
+    Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment
+    pinned to one for chat gets the default Responses version unless it sets responses_api_version.
+    """
+    api_version: Final = _str_param(litellm_params, "api_version")
+    if api_base is not None and "api-version" in httpx.URL(api_base).params:
+        return api_version
+    responses_api_version: Final = _str_param(litellm_params, "responses_api_version")
+    if responses_api_version is not None:
+        return responses_api_version
+    if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version):
+        return AZURE_DEFAULT_RESPONSES_API_VERSION
+    return api_version
+
+
+def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
+    return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
+
+
 class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     # Parameters not supported by Azure Responses API
     AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         - A complete URL string, e.g.,
         "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
         """
-        from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
-
+        resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
         return BaseAzureLLM.get_base_azure_url(
-            api_base=api_base,
-            litellm_params=litellm_params,
+            api_base=resolved_api_base,
+            litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
             route="/openai/responses",
             default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
         )

litellm/llms/azure_ai/chat/transformation.py

@@ -1,6 +1,7 @@
 import copy
 import enum
 import re
+from collections.abc import AsyncIterator, Iterator
 from typing import TYPE_CHECKING, Final, cast
 
 import httpx
@@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import (
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
 from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
-from litellm.llms.openai.openai import OpenAIConfig
+from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
 from litellm.llms.xai.chat.transformation import XAIChatConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import ModelResponse, ProviderField
+from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
 from litellm.utils import add_path_to_api_base, supports_tool_choice
 
 if TYPE_CHECKING:
@@ -45,6 +46,18 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
 )
 
 
+def _is_grok_model(model: str) -> bool:
+    return "grok" in model
+
+
+class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
+    def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream:  # mutable-ok: mirrors override contract
+        usage: Final = chunk.get("usage")
+        if _is_grok_model(str(chunk.get("model") or "")) and isinstance(usage, dict):
+            XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
+        return super().chunk_parser(chunk)
+
+
 class AzureAIGPT5Config(OpenAIGPT5Config):
     @classmethod
     def _model_map_lookup_name(cls, model: str) -> str:
@@ -114,7 +127,7 @@ class AzureAIStudioConfig(OpenAIConfig):
         """
         Check if the model supports stop tokens.
         """
-        if "grok" in model:
+        if _is_grok_model(model):
             # Reuse Xai method for Grok model
             xai_config: Final = XAIChatConfig()
             return xai_config.supports_stop_reason(model)
@@ -319,7 +332,7 @@ class AzureAIStudioConfig(OpenAIConfig):
         json_mode: bool | None = None,
     ) -> ModelResponse:
         model_response.model = f"azure_ai/{model}"
-        return super().transform_response(
+        response: Final = super().transform_response(
             model=model,
             raw_response=raw_response,
             model_response=model_response,
@@ -332,6 +345,21 @@ class AzureAIStudioConfig(OpenAIConfig):
             api_key=api_key,
             json_mode=json_mode,
         )
+        if _is_grok_model(model):
+            XAIChatConfig.fold_reasoning_tokens_into_completion(response)
+        return response
+
+    def get_model_response_iterator(
+        self,
+        streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
+        sync_stream: bool,
+        json_mode: bool | None = False,
+    ) -> "AzureAIChatCompletionResponseIterator":
+        return AzureAIChatCompletionResponseIterator(
+            streaming_response=streaming_response,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+        )
 
     def should_retry_llm_api_inside_llm_translation_on_http_error(
         self, e: httpx.HTTPStatusError, litellm_params: dict

litellm/llms/base_llm/containers/transformation.py

@@ -81,6 +81,7 @@ class BaseContainerConfig(ABC):
         self,
         headers: dict,
         api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
         return {}
 

litellm/llms/base_llm/decisions/transformation.py

@@ -495,6 +495,11 @@ class BaseDecisionsConfig(ABC):
     def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
         return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
 
+    def resolve_credential(
+        self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
+    ) -> str | None:
+        return api_key
+
     @staticmethod
     def _first_secret(names: tuple[str, ...]) -> str | None:
         return next((value for value in (get_secret_str(name) for name in names) if value), None)

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -4,7 +4,7 @@ import json
 from collections.abc import Callable, Iterator, Mapping, Sequence
 from typing import Final, TypeVar, cast  # noqa: TID251  # a rebuilt chat row has no typed constructor across roles
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
 from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
     return flat_name if isinstance(flat_name, str) else None
 
 
+_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
+_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
+
+
+def list_field(data: object, key: str) -> Sequence[object]:
+    try:
+        return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
+    except ValidationError:
+        return ()
+
+
 def anthropic_tool_names(tool: object) -> tuple[str, ...]:
     """Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
     ``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -32,6 +32,7 @@ from litellm.llms.bedrock.common_utils import (
     tools_without_eager_input_streaming,
 )
 from litellm.types.llms.anthropic import (
+    ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
     ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
     AnthropicThinkingParam,
@@ -270,7 +271,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
         is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking)
 
-        user_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
+        client_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
+        user_beta_set: Final = (
+            client_beta_set
+            if "safeguards" in optional_params
+            else client_beta_set - {ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value}
+        )
         beta_set: Final = set(user_beta_set)
         auto_betas: Final = self.get_anthropic_beta_list(
             model=model,

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -90,6 +90,9 @@ class AmazonAnthropicClaudeMessagesConfig(
     def beta_headers_provider(self) -> str:
         return self.custom_llm_provider or "bedrock"
 
+    def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
+        return True
+
     BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
 
     def get_error_class(
@@ -552,8 +555,17 @@ class AmazonAnthropicClaudeMessagesConfig(
         if anthropic_model_info.is_eager_input_streaming_used(tools):
             beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
 
+        dangerous_tool_use_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
         if anthropic_messages_optional_request_params.get("safeguards") is not None:
-            beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
+            beta_set.add(dangerous_tool_use_beta)
+        elif dangerous_tool_use_beta in beta_set and self.drops_dangerous_tool_use_beta_without_safeguards():
+            beta_set.discard(dangerous_tool_use_beta)
+            verbose_logger.debug(
+                "Bedrock Invoke: dropping %s from client headers because the request has no safeguards. Bedrock "
+                "answers it with 'invalid beta flag' next to structured outputs before Opus 4.7, and on Sonnet 4.5, "
+                "Opus 4.5, and Opus 4.6 even alone.",
+                dangerous_tool_use_beta,
+            )
 
         self._filter_context_management_for_bedrock_invoke(
             anthropic_messages_request=anthropic_messages_request,

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -67,6 +67,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
     def should_filter_anthropic_beta_headers(self) -> bool:
         return False
 
+    def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
+        return False
+
     def get_complete_url(
         self,
         api_base: str | None,

litellm/llms/custom_httpx/container_handler.py

@@ -5,6 +5,7 @@ This module provides a single generic handler that can process any container fil
 endpoint defined in endpoints.json, eliminating the need for individual handler methods.
 """
 
+import asyncio
 import json
 from collections.abc import Coroutine, Mapping, Sequence
 from pathlib import Path
@@ -236,6 +237,7 @@ def _request_headers(
     return container_provider_config.validate_environment(
         headers=extra_headers or {},
         api_key=litellm_params.get("api_key", None),
+        litellm_params=litellm_params,
     )
 
 
@@ -435,7 +437,7 @@ class GenericContainerHandler:
         http_client: Final = _async_http_client(client, litellm_params)
 
         # Build request
-        headers = _request_headers(container_provider_config, extra_headers, litellm_params)
+        headers = await asyncio.to_thread(_request_headers, container_provider_config, extra_headers, litellm_params)
         if extra_headers:
             headers.update(extra_headers)
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -7129,7 +7129,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = image_edit_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            image_edit_provider_config.validate_environment,
             api_key=litellm_params.api_key,
             headers=image_edit_optional_request_params.get("extra_headers", {}) or {},
             model=model,
@@ -7588,7 +7589,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_generation_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_generation_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=video_generation_optional_request_params.get("extra_headers", {}) or {},
             model=model,
@@ -7783,7 +7785,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_content_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_content_provider_config.validate_environment,
             headers=extra_headers or {},
             model="",
             api_key=api_key,
@@ -7963,7 +7966,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_remix_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_remix_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8126,7 +8130,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8266,7 +8271,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8438,7 +8444,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8624,7 +8631,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8756,7 +8764,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_list_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_list_provider_config.validate_environment,
             api_key=api_key,
             headers=extra_headers or {},
             model="",
@@ -8837,7 +8846,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_delete_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_delete_provider_config.validate_environment,
             api_key=api_key,
             headers=extra_headers or {},
             model="",
@@ -9021,7 +9031,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_status_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_status_provider_config.validate_environment,
             api_key=api_key,
             headers=extra_headers or {},
             model="",
@@ -9124,6 +9135,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         # Add Content-Type header for JSON requests
@@ -9199,9 +9211,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         # Add Content-Type header for JSON requests
@@ -9295,6 +9309,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9371,9 +9386,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9462,6 +9479,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9538,9 +9556,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9631,6 +9651,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9707,9 +9728,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9805,6 +9828,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9883,9 +9907,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9974,6 +10000,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10049,9 +10076,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10169,8 +10198,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10412,8 +10443,10 @@ class BaseLLMHTTPHandler:
 
         vector_store_provider_config.validate_create_vector_store()
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10543,8 +10576,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10665,8 +10700,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10807,8 +10844,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10952,8 +10991,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -11070,8 +11111,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -11212,8 +11255,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11353,8 +11398,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11484,8 +11531,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11621,8 +11670,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11764,8 +11815,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -39,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
     effective_skip_tool_message_for_guardrail,
+    list_field,
     merge_guardrailed_scoped_messages,
     merge_returned_tools_into_request_tools,
     openai_tool_name,
@@ -156,7 +157,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices]
             # Pass tools (function definitions) to the guardrail
             tools: Final = data.get("tools")
-            if tools and not scan_only_tool_results:
+            if isinstance(tools, list) and tools and not scan_only_tool_results:
                 inputs["tools"] = tools
             # Include model information if available
             model: Final = data.get("model")
@@ -258,19 +259,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             return "no scannable content after message scoping"
         return None if images else "no scannable content"
 
-    def extract_request_tool_names(self, data: dict) -> list[str]:
+    def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
         """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name,
         and the tools a replayed tool_search_output loads)."""
         names: Final[list[str]] = []
-        for tool in data.get("tools") or []:
+        for tool in list_field(data, "tools"):
             if isinstance(tool, dict) and tool.get("type") == "function":
                 fn = tool.get("function")
                 if isinstance(fn, dict) and fn.get("name"):
                     names.append(str(fn["name"]))
-        for fn in data.get("functions") or []:
+        for fn in list_field(data, "functions"):
             if isinstance(fn, dict) and fn.get("name"):
                 names.append(str(fn["name"]))
-        return [*names, *replayed_tool_search_tool_names(cast(object, data))]  # cast-ok: upcast, read field by field
+        return [*names, *replayed_tool_search_tool_names(data)]
 
     def _extract_inputs(
         self,

litellm/llms/openai/common_utils.py

@@ -407,6 +407,10 @@ class BaseOpenAILLM:
             http2=http2_enabled(),
         )
 
+    @classmethod
+    def get_sync_http_client(cls) -> httpx.Client | None:
+        return cls._get_sync_http_client()
+
 
 class OpenAICredentials(NamedTuple):
     api_base: str

litellm/llms/openai/completion/handler.py

@@ -1,9 +1,7 @@
 import json
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
 
-from openai import AsyncOpenAI, OpenAI
-
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@@ -12,7 +10,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
 from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
 from litellm.utils import ProviderConfigManager
 
-from ..common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
+from ..common_utils import BaseOpenAILLM, OpenAIError
+from ..workload_identity import build_async_openai_client, build_openai_client
 from .transformation import OpenAITextCompletionConfig
 
 
@@ -43,7 +42,7 @@ class OpenAITextCompletion(BaseLLM):
         print_verbose: Callable | None = None,
         api_base: str | None = None,
         acompletion: bool = False,
-        litellm_params=None,
+        litellm_params: Mapping[str, object] | None = None,
         logger_fn=None,
         client=None,
         organization: str | None = None,
@@ -95,6 +94,7 @@ class OpenAITextCompletion(BaseLLM):
                         max_retries=max_retries,
                         client=client,
                         organization=organization,
+                        litellm_params=litellm_params,
                     )
                 else:
                     return self.acompletion(
@@ -109,6 +109,7 @@ class OpenAITextCompletion(BaseLLM):
                         max_retries=max_retries,
                         organization=organization,
                         client=client,
+                        litellm_params=litellm_params,
                     )
             elif optional_params.get("stream", False):
                 return self.streaming(
@@ -123,19 +124,22 @@ class OpenAITextCompletion(BaseLLM):
                     max_retries=max_retries,
                     client=client,
                     organization=organization,
+                    litellm_params=litellm_params,
                 )
             else:
-                if client is None:
-                    openai_client = OpenAI(
+                openai_client: Final = (
+                    client
+                    if client is not None
+                    else build_openai_client(
                         api_key=api_key,
-                        base_url=api_base,
-                        http_client=litellm.client_session or OpenAIHTTPClient(),
+                        api_base=api_base,
                         timeout=timeout,
                         max_retries=max_retries,
                         organization=organization,
+                        litellm_params=litellm_params,
+                        static_key_http_client_factory=lambda: litellm.client_session,
                     )
-                else:
-                    openai_client = client
+                )
 
                 raw_response: Final = openai_client.completions.with_raw_response.create(**data)
                 response: Final = raw_response.parse()
@@ -175,19 +179,22 @@ class OpenAITextCompletion(BaseLLM):
         max_retries: int,
         organization: str | None = None,
         client=None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         try:
-            if client is None:
-                openai_aclient = AsyncOpenAI(
+            openai_aclient: Final = (
+                client
+                if client is not None
+                else build_async_openai_client(
                     api_key=api_key,
-                    base_url=api_base,
-                    http_client=BaseOpenAILLM.get_async_http_client(),
+                    api_base=api_base,
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
+                    litellm_params=litellm_params,
+                    static_key_http_client_factory=BaseOpenAILLM.get_async_http_client,
                 )
-            else:
-                openai_aclient = client
+            )
 
             raw_response: Final = await openai_aclient.completions.with_raw_response.create(**data)
             response: Final = raw_response.parse()
@@ -228,18 +235,21 @@ class OpenAITextCompletion(BaseLLM):
         max_retries=None,
         client=None,
         organization=None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
-        if client is None:
-            openai_client = OpenAI(
+        openai_client: Final = (
+            client
+            if client is not None
+            else build_openai_client(
                 api_key=api_key,
-                base_url=api_base,
-                http_client=litellm.client_session or OpenAIHTTPClient(),
+                api_base=api_base,
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
+                static_key_http_client_factory=lambda: litellm.client_session,
             )
-        else:
-            openai_client = client
+        )
 
         try:
             raw_response: Final = openai_client.completions.with_raw_response.create(**data)
@@ -285,18 +295,21 @@ class OpenAITextCompletion(BaseLLM):
         api_base: str | None = None,
         client=None,
         organization=None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
-        if client is None:
-            openai_client = AsyncOpenAI(
+        openai_client: Final = (
+            client
+            if client is not None
+            else build_async_openai_client(
                 api_key=api_key,
-                base_url=api_base,
-                http_client=litellm.aclient_session or OpenAIAsyncHTTPClient(),
+                api_base=api_base,
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
+                static_key_http_client_factory=lambda: litellm.aclient_session,
             )
-        else:
-            openai_client = client
+        )
 
         raw_response: Final = await openai_client.completions.with_raw_response.create(**data)
         response: Final = raw_response.parse()

litellm/llms/openai/containers/transformation.py

@@ -23,6 +23,7 @@ from litellm.types.containers.main import (
 from litellm.types.router import GenericLiteLLMParams
 
 from ...base_llm.containers.transformation import BaseContainerConfig
+from ..workload_identity import resolve_openai_bearer_token
 from .utils import join_container_api_base_path
 
 if TYPE_CHECKING:
@@ -104,13 +105,15 @@ class OpenAIContainerConfig(BaseContainerConfig):
         self,
         headers: dict,
         api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            },
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=api_key or params.api_key,
+            api_base=params.api_base,
+            litellm_params=params.model_dump(exclude_none=True),
         )
+        headers.update({"Authorization": f"Bearer {bearer_token}"})
         return headers
 
     def get_complete_url(

litellm/llms/openai/decisions/transformation.py

@@ -6,6 +6,7 @@ from typing_extensions import assert_never
 
 import litellm
 from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.decisions import (
     DecisionsImage,
@@ -349,6 +350,11 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
     def resolve_api_key(self, api_key: str | None) -> str | None:
         return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
 
+    def resolve_credential(
+        self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
+    ) -> str | None:
+        return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
+
     def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
         return False
 

litellm/llms/openai/evals/transformation.py

@@ -13,6 +13,7 @@ from litellm.llms.base_llm.evals.transformation import (
     BaseEvalsAPIConfig,
     LiteLLMLoggingObj,
 )
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.types.llms.openai_evals import (
     CancelEvalResponse,
     CancelRunResponse,
@@ -44,24 +45,14 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         return LlmProviders.OPENAI
 
     def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
-        """Add OpenAI-specific headers"""
-        import litellm
-        from litellm.secret_managers.main import get_secret_str
-
-        # Get API key following OpenAI pattern
-        api_key = None
-        if litellm_params:
-            api_key = litellm_params.api_key
-
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-
-        if not api_key:
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
+        )
+        if not bearer_token:
             raise ValueError("OPENAI_API_KEY is required for Evals API")
-
-        # Add required headers
-        headers["Authorization"] = f"Bearer {api_key}"
+        headers["Authorization"] = f"Bearer {bearer_token}"
         headers["Content-Type"] = "application/json"
-
         return headers
 
     def get_complete_url(

litellm/llms/openai/fine_tuning/handler.py

@@ -6,7 +6,7 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 from openai.types.fine_tuning import FineTuningJob
 
 from litellm._logging import verbose_logger
-from litellm.llms.openai.common_utils import OpenAIAsyncHTTPClient, OpenAIHTTPClient
+from litellm.llms.openai.workload_identity import build_async_openai_client, build_openai_client
 from litellm.types.utils import LiteLLMFineTuningJob
 
 _AZURE_STATUS_MAP: Final[Mapping[object, str]] = {
@@ -74,27 +74,27 @@ class OpenAIFineTuningAPI:
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
         _is_async: bool = False,
         api_version: str | None = None,
-        litellm_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
-        received_args: Final = locals()
-        openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client" or k == "_is_async":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            if _is_async is True:
-                openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-            else:
-                openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        if _is_async:
+            return build_async_openai_client(
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     async def acreate_fine_tuning_job(
         self,
@@ -116,6 +116,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -126,6 +127,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -164,6 +166,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -174,6 +177,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -212,6 +216,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
         after: str | None = None,
         limit: int | None = None,
     ):
@@ -224,6 +229,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -263,6 +269,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -273,6 +280,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(

litellm/llms/openai/image_edit/transformation.py

@@ -17,6 +17,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import ImageResponse
 
 from ..common_utils import OpenAIError
+from ..workload_identity import resolve_openai_bearer_token
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -160,12 +161,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
         litellm_params: dict | None = None,
         api_base: str | None = None,
     ) -> dict:
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            }
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=api_key, api_base=api_base, litellm_params=litellm_params
         )
+        headers.update({"Authorization": f"Bearer {bearer_token}"})
         return headers
 
     def get_complete_url(

litellm/llms/openai/image_variations/handler.py

@@ -2,7 +2,7 @@
 OpenAI Image Variations Handler
 """
 
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
 
 import httpx
@@ -14,27 +14,58 @@ from litellm.utils import ProviderConfigManager
 
 from ...base_llm.image_variations.transformation import BaseImageVariationConfig
 from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
-from ..common_utils import OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
+from ..common_utils import OpenAIError
+from ..workload_identity import build_async_openai_client, build_openai_client
 
 
 class OpenAIImageVariationsHandler:
     def get_sync_client(
         self,
         client: OpenAI | None,
-        init_client_params: dict,
-    ):
+        api_key: str | None,
+        api_base: str,
+        timeout: float | httpx.Timeout | None,
+        max_retries: int,
+        organization: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> OpenAI:
         if client is not None:
             return client
-        return OpenAI(**init_client_params, http_client=litellm.client_session or OpenAIHTTPClient())
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+            static_key_http_client_factory=lambda: litellm.client_session,
+        )
 
-    def get_async_client(self, client: AsyncOpenAI | None, init_client_params: dict) -> AsyncOpenAI:
+    def get_async_client(
+        self,
+        client: AsyncOpenAI | None,
+        api_key: str | None,
+        api_base: str,
+        timeout: float | httpx.Timeout | None,
+        max_retries: int,
+        organization: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> AsyncOpenAI:
         if client is not None:
             return client
-        return AsyncOpenAI(**init_client_params, http_client=litellm.aclient_session or OpenAIAsyncHTTPClient())
+        return build_async_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+            static_key_http_client_factory=lambda: litellm.aclient_session,
+        )
 
     async def async_image_variations(
         self,
-        api_key: str,
+        api_key: str | None,
         api_base: str,
         organization: str | None,
         client: AsyncOpenAI | None,
@@ -51,15 +82,15 @@ class OpenAIImageVariationsHandler:
         provider_config: BaseImageVariationConfig,
     ) -> ImageResponse:
         try:
-            init_client_params: Final = {
-                "api_key": api_key,
-                "base_url": api_base,
-                "timeout": timeout,
-                "max_retries": max_retries,
-                "organization": organization,
-            }
-
-            client = self.get_async_client(client=client, init_client_params=init_client_params)
+            client = self.get_async_client(
+                client=client,
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
 
             raw_response: Final = await client.images.with_raw_response.create_variation(**data)
             response: Final = raw_response.parse()
@@ -103,7 +134,7 @@ class OpenAIImageVariationsHandler:
     def image_variations(
         self,
         model_response: ImageResponse,
-        api_key: str,
+        api_key: str | None,
         api_base: str,
         model: str | None,
         image: FileTypes,
@@ -167,15 +198,15 @@ class OpenAIImageVariationsHandler:
                     litellm_params=litellm_params,
                 )
 
-            init_client_params: Final = {
-                "api_key": api_key,
-                "base_url": api_base,
-                "timeout": timeout,
-                "max_retries": max_retries,
-                "organization": organization,
-            }
-
-            client = self.get_sync_client(client=client, init_client_params=init_client_params)
+            client = self.get_sync_client(
+                client=client,
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
 
             raw_response: Final = client.images.with_raw_response.create_variation(**json_data)
             response: Final = raw_response.parse()

litellm/llms/openai/openai.py

@@ -52,16 +52,19 @@ from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig
 from .chat.o_series_transformation import OpenAIOSeriesConfig
 from .common_utils import (
     BaseOpenAILLM,
-    OpenAIAsyncHTTPClient,
     OpenAIError,
-    OpenAIHTTPClient,
     build_output_token_limit_response,
     drop_params_from_unprocessable_entity_error,
     is_openai_backed_api_base,
     is_output_token_limit_error,
     model_dump_provider_response,
 )
-from .workload_identity import resolve_openai_workload_identity_config
+from .workload_identity import (
+    build_async_openai_client,
+    build_openai_client,
+    openai_static_api_key,
+    resolve_openai_workload_identity_config,
+)
 
 openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
 openAIGPT5Config: Final = OpenAIGPT5Config()
@@ -425,7 +428,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     )
                     if workload_identity_config is not None
                     else AsyncOpenAI(
-                        api_key=api_key,
+                        api_key=openai_static_api_key(api_key),
                         base_url=api_base,
                         http_client=async_http_client,
                         timeout=timeout,
@@ -447,7 +450,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     )
                     if workload_identity_config is not None
                     else OpenAI(
-                        api_key=api_key,
+                        api_key=openai_static_api_key(api_key),
                         base_url=api_base,
                         http_client=sync_http_client,
                         timeout=timeout,
@@ -1455,10 +1458,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         max_retries=None,
         organization: str | None = None,
         headers: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         try:
             openai_aclient: Final = self._get_openai_client(
                 is_async=True,
+                litellm_params=litellm_params,
                 api_key=api_key,
                 api_base=api_base,
                 timeout=timeout,
@@ -1520,6 +1525,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         aimg_generation=None,
         organization: str | None = None,
         headers: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> ImageResponse:
         data = {}
         try:
@@ -1541,10 +1547,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     max_retries=max_retries,
                     organization=organization,
                     headers=headers,
+                    litellm_params=litellm_params,
                 )
 
             openai_client: Final[OpenAI] = self._get_openai_client(
                 is_async=False,
+                litellm_params=litellm_params,
                 api_key=api_key,
                 api_base=api_base,
                 timeout=timeout,
@@ -1622,6 +1630,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         aspeech: bool | None = None,
         client=None,
         shared_session: Optional["ClientSession"] = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> HttpxBinaryResponseContent:
         if aspeech is not None and aspeech is True:
             return self.async_audio_speech(
@@ -1638,6 +1647,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 logging_obj=logging_obj,
                 client=client,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             )
 
         openai_client: Final = self._get_openai_client(
@@ -1648,6 +1658,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             max_retries=max_retries,
             client=client,
             shared_session=shared_session,
+            litellm_params=litellm_params,
         )
 
         sync_client: Final = cast(OpenAI, openai_client)
@@ -1685,6 +1696,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         logging_obj: LiteLLMLoggingObj,
         client=None,
         shared_session: Optional["ClientSession"] = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> HttpxBinaryResponseContent:
         openai_client: Final = cast(
             AsyncOpenAI,
@@ -1696,6 +1708,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 max_retries=max_retries,
                 client=client,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             ),
         )
 
@@ -1742,26 +1755,27 @@ class OpenAIFilesAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
         _is_async: bool = False,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | None:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        openai_client: OpenAI | AsyncOpenAI | None = None
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client" or k == "_is_async":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            if _is_async is True:
-                openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-            else:
-                openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        if _is_async:
+            return build_async_openai_client(
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     async def acreate_file(
         self,
@@ -1781,6 +1795,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAIFileObject | Coroutine[None, None, OpenAIFileObject]:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1790,6 +1805,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1823,6 +1839,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1832,6 +1849,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1888,6 +1906,7 @@ class OpenAIFilesAPI(BaseLLM):
         organization: str | None,
         chunk_size: int = 1024 * 1024,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> FileContentStreamingResult:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1897,6 +1916,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1951,6 +1971,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1960,6 +1981,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1997,6 +2019,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2006,6 +2029,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2046,6 +2070,7 @@ class OpenAIFilesAPI(BaseLLM):
         organization: str | None,
         purpose: str | None = None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2055,6 +2080,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2100,26 +2126,27 @@ class OpenAIBatchesAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
         _is_async: bool = False,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | None:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        openai_client: OpenAI | AsyncOpenAI | None = None
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client" or k == "_is_async":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            if _is_async is True:
-                openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-            else:
-                openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        if _is_async:
+            return build_async_openai_client(
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     async def acreate_batch(
         self,
@@ -2139,6 +2166,7 @@ class OpenAIBatchesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2148,6 +2176,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2183,6 +2212,7 @@ class OpenAIBatchesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2192,6 +2222,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2226,6 +2257,7 @@ class OpenAIBatchesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2235,6 +2267,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2275,6 +2308,7 @@ class OpenAIBatchesAPI(BaseLLM):
         after: str | None = None,
         limit: int | None = None,
         client: OpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2284,6 +2318,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2312,22 +2347,19 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     def async_get_openai_client(
         self,
@@ -2337,22 +2369,19 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AsyncOpenAI:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        return build_async_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     ### ASSISTANTS ###
 
@@ -2368,6 +2397,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         limit: int | None = 20,
         before: str | None = None,
         after: str | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AsyncCursorPage[Assistant]:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2375,6 +2406,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
         request_params: Final = {
@@ -2402,6 +2434,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         aget_assistants: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, AsyncCursorPage[Assistant]]:
         ...
 
@@ -2415,6 +2449,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         aget_assistants: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> SyncCursorPage[Assistant]: 
         ...
 
@@ -2433,7 +2469,9 @@ class OpenAIAssistantsAPI(BaseLLM):
         limit: int | None = 20,
         before: str | None = None,
         after: str | None = None,
-    ):
+        *,
+        litellm_params: Mapping[str, object] | None = None,
+    ) -> Coroutine[None, None, AsyncCursorPage[Assistant]] | SyncCursorPage[Assistant]:
         if aget_assistants is not None and aget_assistants is True:
             return self.async_get_assistants(
                 api_key=api_key,
@@ -2441,6 +2479,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2449,6 +2488,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2476,6 +2516,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         create_assistant_data: dict,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Assistant:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2483,6 +2525,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2500,6 +2543,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         create_assistant_data: dict,
         client=None,
         async_create_assistants=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if async_create_assistants is not None and async_create_assistants is True:
             return self.async_create_assistants(
@@ -2508,6 +2553,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
                 create_assistant_data=create_assistant_data,
             )
@@ -2517,6 +2563,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2533,6 +2580,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         assistant_id: str,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AssistantDeleted:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2540,6 +2589,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2557,6 +2607,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         client=None,
         async_delete_assistants=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if async_delete_assistants is not None and async_delete_assistants is True:
             return self.async_delete_assistant(
@@ -2565,6 +2617,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
                 assistant_id=assistant_id,
             )
@@ -2574,6 +2627,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2592,6 +2646,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAIMessage:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2599,6 +2655,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2629,6 +2686,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         a_add_message: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, OpenAIMessage]:
         ...
 
@@ -2644,6 +2703,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         a_add_message: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAIMessage: 
         ...
 
@@ -2660,6 +2721,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client=None,
         a_add_message: bool | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if a_add_message is not None and a_add_message is True:
             return self.a_add_message(
@@ -2670,6 +2733,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2678,6 +2742,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2703,6 +2768,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AsyncCursorPage[OpenAIMessage]:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2710,6 +2777,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2730,6 +2798,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         aget_messages: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
         ...
 
@@ -2744,6 +2814,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         aget_messages: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> SyncCursorPage[OpenAIMessage]: 
         ...
 
@@ -2759,6 +2831,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client=None,
         aget_messages=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if aget_messages is not None and aget_messages is True:
             return self.async_get_messages(
@@ -2768,6 +2842,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2776,6 +2851,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2795,6 +2871,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2802,6 +2880,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2834,6 +2913,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
         client: AsyncOpenAI | None,
         acreate_thread: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, Thread]:
         ...
 
@@ -2849,6 +2930,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
         client: OpenAI | None,
         acreate_thread: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread: 
         ...
 
@@ -2865,6 +2948,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
         client=None,
         acreate_thread=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         """
         Here's an example:
@@ -2884,6 +2969,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
                 messages=messages,
             )
@@ -2893,6 +2979,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2920,6 +3007,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2927,6 +3016,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2952,6 +3042,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         aget_thread: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, Thread]:
         ...
 
@@ -2966,6 +3058,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         aget_thread: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread: 
         ...
 
@@ -2981,6 +3075,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client=None,
         aget_thread=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if aget_thread is not None and aget_thread is True:
             return self.async_get_thread(
@@ -2990,6 +3086,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2998,6 +3095,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -3031,6 +3129,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Run:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -3038,6 +3138,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -3142,6 +3243,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         client,
         arun_thread: Literal[True], 
         event_handler: AssistantEventHandler | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, Run]:
         ...
 
@@ -3164,6 +3267,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         client,
         arun_thread: Literal[False] | None, 
         event_handler: AssistantEventHandler | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Run: 
         ...
 
@@ -3187,6 +3292,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         client=None,
         arun_thread=None,
         event_handler: AssistantEventHandler | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if arun_thread is not None and arun_thread is True:
             if stream is not None and stream is True:
@@ -3196,6 +3303,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
+                    litellm_params=litellm_params,
                     client=client,
                 )
                 return self.async_run_thread_stream(
@@ -3223,6 +3331,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -3231,6 +3340,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 

litellm/llms/openai/responses/count_tokens/token_counter.py

@@ -27,6 +27,7 @@ from litellm.llms.openai.responses.count_tokens.transformation import (
 from litellm.llms.openai.workload_identity import (
     OpenAIWorkloadIdentityConfig,
     get_workload_identity_bearer_token,
+    openai_static_api_key,
     resolve_openai_workload_identity_config,
 )
 from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
@@ -35,13 +36,18 @@ from litellm.types.utils import CountContextManagement, LlmProviders, TokenCount
 openai_count_tokens_handler: Final = OpenAICountTokensHandler()
 
 
-def _workload_identity_config(litellm_params: Mapping[str, object]) -> OpenAIWorkloadIdentityConfig | None:
+def _workload_identity_config(
+    litellm_params: Mapping[str, object],
+) -> OpenAIWorkloadIdentityConfig | OpenAIError | None:
     api_base: Final = litellm_params.get("api_base")
-    return resolve_openai_workload_identity_config(
-        api_key=None,
-        api_base=api_base if isinstance(api_base, str) else None,
-        litellm_params=litellm_params,
-    )
+    try:
+        return resolve_openai_workload_identity_config(
+            api_key=None,
+            api_base=api_base if isinstance(api_base, str) else None,
+            litellm_params=litellm_params,
+        )
+    except OpenAIError as e:
+        return e
 
 
 class OpenAITokenCounter(BaseTokenCounter):
@@ -126,7 +132,18 @@ class OpenAITokenCounter(BaseTokenCounter):
         credentials: Final = get_openai_credentials(
             api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key")
         )
-        credential: Final = credentials.api_key or _workload_identity_config(litellm_params)
+        credential: Final = openai_static_api_key(credentials.api_key) or _workload_identity_config(litellm_params)
+
+        if isinstance(credential, OpenAIError):
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=credential.message,
+                status_code=credential.status_code,
+            )
 
         if credential is None:
             return TokenCountResponse(

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -55,6 +55,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
 from litellm.llms.base_llm.guardrail_translation.utils import (
     blocked_responses_stream_usage,
     effective_skip_system_message_for_guardrail,
+    list_field,
     merge_guardrailed_scoped_messages,
     role_out_of_guardrail_scope,
     scoped_structured_message_indices,
@@ -764,11 +765,11 @@ class OpenAIResponsesHandler(BaseTranslation):
             data.get("input"), data.get("instructions"), structured_messages, merged
         )
 
-    def extract_request_tool_names(self, data: dict) -> list[str]:
+    def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
         """Extract tool names from Responses API request (tools[].name for function
         and custom, tools[].server_label for mcp)."""
         names: Final[list[str]] = []
-        for tool in data.get("tools") or []:
+        for tool in list_field(data, "tools"):
             if not isinstance(tool, dict):
                 continue
             if tool.get("type") in ("function", "custom") and tool.get("name"):

litellm/llms/openai/transcriptions/handler.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final, Optional, cast
 
 import httpx
@@ -97,6 +98,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
                 max_retries=max_retries,
                 logging_obj=logging_obj,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             )
 
         openai_client: Final[OpenAI] = self._get_openai_client(
@@ -106,6 +108,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
             timeout=timeout,
             max_retries=max_retries,
             client=client,
+            litellm_params=litellm_params,
         )
 
         ## LOGGING
@@ -159,6 +162,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
         client=None,
         max_retries=None,
         shared_session: Optional["ClientSession"] = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         try:
             openai_aclient: Final[AsyncOpenAI] = self._get_openai_client(
@@ -169,6 +173,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
                 max_retries=max_retries,
                 client=client,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             )
 
             ## LOGGING

litellm/llms/openai/vector_store_files/transformation.py

@@ -8,6 +8,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store_files.transformation import (
     BaseVectorStoreFilesConfig,
 )
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.vector_store_files import (
@@ -70,11 +71,13 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
         headers: dict[str, str],
         litellm_params: GenericLiteLLMParams | None,
     ) -> dict[str, str]:
-        litellm_params = litellm_params or GenericLiteLLMParams()
-        api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
+        )
         headers.update(
             {
-                "Authorization": f"Bearer {api_key}",
+                "Authorization": f"Bearer {bearer_token}",
                 "Content-Type": "application/json",
             }
         )

litellm/llms/openai/vector_stores/transformation.py

@@ -5,6 +5,7 @@ import httpx
 import litellm
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.vector_stores import (
@@ -48,11 +49,13 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
         }
 
     def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
-        litellm_params = litellm_params or GenericLiteLLMParams()
-        api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
+        )
         headers.update(
             {
-                "Authorization": f"Bearer {api_key}",
+                "Authorization": f"Bearer {bearer_token}",
                 "Content-Type": "application/json",
             }
         )

litellm/llms/openai/videos/transformation.py

@@ -8,11 +8,10 @@ from urllib.parse import quote
 import httpx
 from httpx._types import FileContent, FileTypes, RequestFiles
 
-import litellm
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
 from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils
-from litellm.secret_managers.main import get_secret_str
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.types.llms.openai import CreateVideoRequest
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.videos.main import (
@@ -78,16 +77,13 @@ class OpenAIVideoConfig(BaseVideoConfig):
         api_key: str | None = None,
         litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
-        # Use api_key from litellm_params if available, otherwise fall back to other sources
-        if litellm_params and litellm_params.api_key:
-            api_key = api_key or litellm_params.api_key
-
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            }
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=api_key or params.api_key,
+            api_base=params.api_base,
+            litellm_params=params.model_dump(exclude_none=True),
         )
+        headers.update({"Authorization": f"Bearer {bearer_token}"})
         return headers
 
     def get_complete_url(

litellm/llms/openai/workload_identity.py

@@ -6,10 +6,22 @@ from functools import lru_cache
 from typing import TYPE_CHECKING, Final
 from urllib.parse import urlparse
 
-import litellm
-from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
+import httpx
+from openai import DEFAULT_MAX_RETRIES as OPENAI_SDK_DEFAULT_MAX_RETRIES
+from openai import NOT_GIVEN, AsyncOpenAI, NotGiven, OpenAI
 
-from .common_utils import OpenAIError, is_openai_backed_api_base
+import litellm
+from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
+from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
+from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
+
+from .common_utils import (
+    BaseOpenAILLM,
+    OpenAIAsyncHTTPClient,
+    OpenAIError,
+    OpenAIHTTPClient,
+    is_openai_backed_api_base,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -17,6 +29,11 @@ if TYPE_CHECKING:
     from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
 
 OPENAI_WIF_CLIENT_ID: Final = "litellm"
+_IDENTITY_PARAM_ENV_NAMES: Final = (
+    ("openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"),
+    ("openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID"),
+    ("openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE"),
+)
 _SDK_UPGRADE_MESSAGE: Final = (
     "OpenAI workload identity federation requires openai>=2.32.0. "
     "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
@@ -51,17 +68,25 @@ def resolve_openai_workload_identity_config(
     )
     if static_api_key is not None:
         return None
-    effective_api_base: Final = (
-        api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
-    )
-    if not _targets_openai_api(effective_api_base):
+    return _configured_openai_workload_identity(api_base=api_base, litellm_params=litellm_params)
+
+
+def _effective_openai_api_base(api_base: str | None) -> str | None:
+    return api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
+
+
+def _configured_openai_workload_identity(
+    api_base: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> OpenAIWorkloadIdentityConfig | None:
+    if not _targets_openai_api(_effective_openai_api_base(api_base)):
         return None
-    identity_provider_id: Final = _config_value(
-        litellm_params, "openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"
+    identity_provider_id, service_account_id, token_file = (
+        _config_value(litellm_params, param_key, env_name) for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES
     )
-    service_account_id: Final = _config_value(litellm_params, "openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID")
-    token_file: Final = _config_value(litellm_params, "openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE")
     if identity_provider_id is None or service_account_id is None or token_file is None:
+        if _carries_deployment_identity(litellm_params):
+            raise OpenAIError(status_code=401, message=_incomplete_identity_message(litellm_params))
         return None
     return OpenAIWorkloadIdentityConfig(
         identity_provider_id=identity_provider_id,
@@ -81,11 +106,200 @@ async def get_workload_identity_bearer_token_for_api_base(api_base: str) -> str
     return await _workload_identity_auth(config).get_token_async()
 
 
+def openai_static_api_key(api_key: str | None) -> str | None:
+    return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+
+
+def resolve_openai_bearer_token(
+    api_key: str | None,
+    api_base: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> str | None:
+    static_api_key: Final = openai_static_api_key(api_key)
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=static_api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    if workload_identity_config is None:
+        return static_api_key
+    return get_workload_identity_bearer_token(workload_identity_config)
+
+
+async def resolve_openai_bearer_token_async(
+    static_api_key: str | None,
+    api_base: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> str | None:
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=static_api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    if workload_identity_config is None:
+        return static_api_key
+    return await _workload_identity_auth(workload_identity_config).get_token_async()
+
+
+def _carries_deployment_identity(litellm_params: Mapping[str, object] | None) -> bool:
+    if litellm_params is None:
+        return False
+    return any(_param_str(litellm_params, key) is not None for key in OPENAI_WIF_KWARGS_KEYS)
+
+
+def _incomplete_identity_message(litellm_params: Mapping[str, object] | None) -> str:
+    missing: Final = ", ".join(
+        param_key
+        for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES
+        if _config_value(litellm_params, param_key, env_name) is None
+    )
+    return (
+        f"OpenAI workload identity federation on this deployment is missing {missing}. "
+        "Set openai_identity_provider_id, openai_service_account_id, and openai_identity_token_file "
+        "on the deployment, or the OPENAI_IDENTITY_PROVIDER_ID / OPENAI_SERVICE_ACCOUNT_ID / "
+        "OPENAI_IDENTITY_TOKEN_FILE env vars they fall back to."
+    )
+
+
+def _param_str(litellm_params: Mapping[str, object], key: str) -> str | None:
+    value: Final = litellm_params.get(key)
+    return value if isinstance(value, str) and value else None
+
+
 def _config_value(litellm_params: Mapping[str, object] | None, param_key: str, env_name: str) -> str | None:
-    param_value: Final = litellm_params.get(param_key) if litellm_params is not None else None
-    if isinstance(param_value, str) and param_value:
-        return param_value
-    return normalize_nonempty_secret_str(get_secret_str(env_name))
+    param_value: Final = _param_str(litellm_params, param_key) if litellm_params is not None else None
+    return param_value or normalize_nonempty_secret_str(get_secret_str(env_name))
+
+
+def build_openai_client(
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    max_retries: int | None = None,
+    organization: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
+    static_key_http_client_factory: Callable[[], httpx.Client | None] | None = None,
+) -> OpenAI:
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    retries: Final = _max_retries_or_sdk_default(max_retries)
+    if workload_identity_config is None:
+        static_key_http_client: Final = (
+            None if static_key_http_client_factory is None else static_key_http_client_factory()
+        )
+        return OpenAI(
+            api_key=openai_static_api_key(api_key),
+            base_url=api_base,
+            timeout=timeout,
+            max_retries=retries,
+            organization=organization,
+            http_client=static_key_http_client or OpenAIHTTPClient(),
+        )
+    cache_params: Final = _client_cache_params(
+        is_async=False,
+        workload_identity_config=workload_identity_config,
+        api_base=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+    )
+    cached: Final = BaseOpenAILLM.get_cached_openai_client(
+        client_initialization_params=cache_params, client_type="openai"
+    )
+    if isinstance(cached, OpenAI):
+        return cached
+    process_http_client: Final = BaseOpenAILLM.get_sync_http_client()
+    client: Final = OpenAI(
+        workload_identity=workload_identity_config.to_sdk_workload_identity(),
+        base_url=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+        http_client=process_http_client,
+    )
+    BaseOpenAILLM.set_cached_openai_client(
+        openai_client=client,
+        client_type="openai",
+        client_initialization_params=cache_params,
+        litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client),
+    )
+    return client
+
+
+def build_async_openai_client(
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    max_retries: int | None = None,
+    organization: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
+    static_key_http_client_factory: Callable[[], httpx.AsyncClient | None] | None = None,
+) -> AsyncOpenAI:
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    retries: Final = _max_retries_or_sdk_default(max_retries)
+    if workload_identity_config is None:
+        static_key_http_client: Final = (
+            None if static_key_http_client_factory is None else static_key_http_client_factory()
+        )
+        return AsyncOpenAI(
+            api_key=openai_static_api_key(api_key),
+            base_url=api_base,
+            timeout=timeout,
+            max_retries=retries,
+            organization=organization,
+            http_client=static_key_http_client or OpenAIAsyncHTTPClient(),
+        )
+    cache_params: Final = _client_cache_params(
+        is_async=True,
+        workload_identity_config=workload_identity_config,
+        api_base=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+    )
+    cached: Final = BaseOpenAILLM.get_cached_openai_client(
+        client_initialization_params=cache_params, client_type="openai"
+    )
+    if isinstance(cached, AsyncOpenAI):
+        return cached
+    process_http_client: Final = BaseOpenAILLM.get_async_http_client()
+    client: Final = AsyncOpenAI(
+        workload_identity=workload_identity_config.to_sdk_workload_identity(),
+        base_url=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+        http_client=process_http_client,
+    )
+    BaseOpenAILLM.set_cached_openai_client(
+        openai_client=client,
+        client_type="openai",
+        client_initialization_params=cache_params,
+        litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client),
+    )
+    return client
+
+
+def _client_cache_params(
+    is_async: bool,
+    workload_identity_config: OpenAIWorkloadIdentityConfig,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None | NotGiven,
+    max_retries: int,
+    organization: str | None,
+) -> dict[str, object]:
+    return {
+        "api_key": None,
+        "is_async": is_async,
+        "workload_identity_config": workload_identity_config,
+        "api_base": api_base,
+        "timeout": timeout,
+        "max_retries": max_retries,
+        "organization": organization,
+    }
+
+
+def _max_retries_or_sdk_default(max_retries: int | None) -> int:
+    return OPENAI_SDK_DEFAULT_MAX_RETRIES if max_retries is None else max_retries
 
 
 def _targets_openai_api(api_base: str | None) -> bool:
@@ -95,7 +309,7 @@ def _targets_openai_api(api_base: str | None) -> bool:
     return parsed.scheme == "https" and is_openai_backed_api_base(api_base)
 
 
-@lru_cache(maxsize=16)
+@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
 def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth:
     sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth()
     return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity())

litellm/main.py

@@ -255,6 +255,7 @@ from .llms.openai.completion.handler import OpenAITextCompletion
 from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler
 from .llms.openai.openai import OpenAIChatCompletion
 from .llms.openai.transcriptions.handler import OpenAIAudioTranscription
+from .llms.openai.workload_identity import build_openai_client
 from .llms.openai_like.chat.handler import OpenAILikeChatHandler
 from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler
 from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
@@ -1212,13 +1213,18 @@ def _register_custom_pricing_for_request(
         litellm.register_model({shared_key: entry}, persist_across_reloads=False)
         return
     litellm.register_model(
-        {
-            deployment_id: entry,
-            shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
-        },
+        {deployment_id: entry},
         persist_across_reloads=False,
         warning_display_name=shared_key,
         custom_llm_provider=custom_llm_provider,
+        add_to_provider_model_sets=False,
+    )
+    litellm.register_model(
+        {shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)},
+        persist_across_reloads=False,
+        warning_display_name=shared_key,
+        custom_llm_provider=custom_llm_provider,
+        add_to_provider_model_sets=False,
     )
 
 
@@ -5785,6 +5791,7 @@ def completion(
                     "github_copilot_user_session",
                     PROVIDER_AFFINITY_HEADER_KWARG_KEY,
                     "fireworks_forward_user_id",
+                    "responses_api_version",
                 )
                 if key in kwargs
             },
@@ -7860,17 +7867,14 @@ def adapter_completion(*, adapter_id: str, **kwargs) -> BaseModel | AdapterCompl
 
 
 def moderation(input: str, model: str | None = None, api_key: str | None = None, **kwargs) -> OpenAIModerationResponse:
-    from litellm.llms.openai.common_utils import OpenAIHTTPClient
-
-    # only supports open ai for now
-    api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-
-    # Extract api_base from kwargs
+    static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
     api_base: Final = kwargs.get("api_base", None)
 
-    openai_client = kwargs.get("client", None)
-    if openai_client is None:
-        openai_client = openai.OpenAI(api_key=api_key, base_url=api_base, http_client=OpenAIHTTPClient())
+    openai_client: Final = kwargs.get("client", None) or build_openai_client(
+        api_key=static_api_key,
+        api_base=api_base,
+        litellm_params=kwargs,
+    )
 
     if model is not None:
         response = openai_client.moderations.create(input=input, model=model)
@@ -7893,8 +7897,7 @@ async def amoderation(
 ) -> OpenAIModerationResponse:
     from openai import AsyncOpenAI
 
-    # only supports open ai for now
-    api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+    static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
     optional_params: Final = GenericLiteLLMParams.model_validate(kwargs)
     litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
     _dynamic_api_base = None
@@ -7920,8 +7923,9 @@ async def amoderation(
         # _get_openai_client maintains in-memory caching logic for OpenAI clients
         _openai_client: AsyncOpenAI = openai_chat_completions.get_openai_client(
             is_async=True,
-            api_key=api_key,
+            api_key=static_api_key,
             api_base=optional_params.api_base or _dynamic_api_base,
+            litellm_params=optional_params.model_dump(exclude_none=True),
         )
     else:
         _openai_client = openai_client
@@ -7941,7 +7945,7 @@ async def amoderation(
         moderation_request: Final = {"input": input, "model": model}
         litellm_logging_obj.pre_call(
             input=input,
-            api_key=api_key,
+            api_key=static_api_key,
             additional_args={
                 "complete_input_dict": moderation_request,
                 "api_base": str(_openai_client.base_url),
@@ -8174,9 +8178,7 @@ def transcription(
             or get_secret("OPENAI_ORGANIZATION")
             or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
         )
-        # set API KEY
-
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
+        static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
         response = openai_audio_transcriptions.audio_transcriptions(
             model=model,
             audio_file=file,
@@ -8188,7 +8190,7 @@ def transcription(
             logging_obj=litellm_logging_obj,
             max_retries=max_retries,
             api_base=api_base,
-            api_key=api_key,
+            api_key=static_api_key,
             provider_config=provider_config,
             litellm_params=litellm_params_dict,
             shared_session=shared_session,
@@ -8510,6 +8512,7 @@ def speech(
             client=client,  # pass AsyncOpenAI, OpenAI client
             aspeech=aspeech,
             shared_session=shared_session,
+            litellm_params=litellm_params_dict,
         )
     elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
         # Check if this is Azure Speech Service (Cognitive Services TTS)

litellm/proxy/_experimental/mcp_server/subscriptions.py

@@ -55,6 +55,7 @@ from litellm.proxy._experimental.mcp_server.catalog import (
 )
 from litellm.proxy._experimental.mcp_server.contracts import OperationContext
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
 from litellm.types.mcp import MCP_LEGACY_VERSIONS
 from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
@@ -300,6 +301,29 @@ async def _listing(event: ServerEvent, context: OperationContext) -> str | None:
     raise MCPError(code=-32603, message="Subscription catalog exceeds the pagination limit")
 
 
+async def _enforce_rate_limit(context: OperationContext, server: MCPServer) -> None:
+    await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, server)  # pyright: ignore[reportPrivateUsage]  # same admission policy as other gateway operations
+
+
+async def _unmetered(_: OperationContext, __: MCPServer) -> None:
+    return None
+
+
+async def _admitted() -> ProxyRateLimitError | None:
+    return None
+
+
+@asynccontextmanager
+async def _listener_admission(targets: tuple[SubscriptionTarget, ...]) -> AsyncGenerator[None]:
+    scope: Final = operations._mcp_server_admission_memo.set(  # pyright: ignore[reportPrivateUsage]  # the listener's own catalog refreshes reuse its admission
+        {target.server.server_id: asyncio.create_task(_admitted()) for target in targets}
+    )
+    try:
+        yield
+    finally:
+        operations._mcp_server_admission_memo.reset(scope)  # pyright: ignore[reportPrivateUsage]  # restore the caller's admission scope
+
+
 async def _recheck_authorization(check: Callable[[], Awaitable[OperationContext]]) -> None:
     while True:
         await asyncio.sleep(15)
@@ -361,7 +385,7 @@ class GatewaySubscriptions:
         if ctx.protocol_version != "2026-07-28":
             raise MCPError(code=-32601, message="Subscriptions require the modern MCP protocol")
         async with self._capacity(owner):
-            return await self._listen(ctx, params, authorize)
+            return await self._listen(ctx, params, authorize, _enforce_rate_limit)
 
     @asynccontextmanager
     async def _capacity(self, owner: str) -> AsyncGenerator[None]:
@@ -414,6 +438,7 @@ class GatewaySubscriptions:
                     ctx,
                     SubscriptionFilter(tools_list_changed=True, prompts_list_changed=True, resources_list_changed=True),
                     authorization,
+                    _unmetered,
                 )
                 if (ready := self._initializing.get(key)) is not None:
                     ready.set()
@@ -427,6 +452,7 @@ class GatewaySubscriptions:
         ctx: ServerRequestContext,
         notifications: SubscriptionFilter,
         authorization: SubscriptionAuthorization,
+        admit: Callable[[OperationContext, MCPServer], Awaitable[None]],
         scope: anyio.CancelScope | None = None,
         *,
         task_status: TaskStatus[None],
@@ -443,7 +469,7 @@ class GatewaySubscriptions:
 
             async with self._capacity(authorization.owner):
                 await self._listen(
-                    ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, ready
+                    ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, admit, ready
                 )
 
     async def legacy_resource(self, ctx: ServerRequestContext, uri: str, *, subscribe: bool) -> None:
@@ -464,6 +490,7 @@ class GatewaySubscriptions:
                     ctx,
                     SubscriptionFilter(resource_subscriptions=[uri]),
                     state.authorization,
+                    _enforce_rate_limit,
                     scope,
                 )
                 state.resources = {**state.resources, uri: scope}
@@ -476,6 +503,7 @@ class GatewaySubscriptions:
         ctx: ServerRequestContext,
         params: SubscriptionsListenRequestParams,
         authorize: Callable[[], Awaitable[OperationContext]],
+        admit: Callable[[OperationContext, MCPServer], Awaitable[None]],
         ready: Callable[[SubscriptionFilter], None] | None = None,
     ) -> SubscriptionsListenResult:
         context: Final = await authorize()
@@ -501,7 +529,7 @@ class GatewaySubscriptions:
             return fresh
 
         async def relay(target: SubscriptionTarget, *, task_status: TaskStatus[SubscriptionFilter]) -> None:
-            await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, target.server)  # pyright: ignore[reportPrivateUsage]  # same admission policy as other gateway operations
+            await admit(context, target.server)
 
             legacy: Final = LegacyNotifications()
 
@@ -567,7 +595,7 @@ class GatewaySubscriptions:
             finally:
                 self._upstream_slots.release()
 
-        async with anyio.create_task_group() as group:
+        async with _listener_admission(targets), anyio.create_task_group() as group:
             honored: Final = tuple(
                 [SubscriptionFilter.model_validate(await group.start(relay, target)) for target in targets]
             )

litellm/proxy/_lazy_openapi_snapshot.json

@@ -14261,6 +14261,19 @@
               "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
               "title": "Fail On Error"
             },
+            "forward_messages_as_sent": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "default": false,
+              "description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.",
+              "title": "Forward Messages As Sent"
+            },
             "guard_name": {
               "anyOf": [
                 {
@@ -16612,6 +16625,19 @@
               "description": "HTTP timeout in seconds for checking attached files. Default: 10.",
               "title": "File Guardrail Timeout"
             },
+            "forward_messages_as_sent": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "default": false,
+              "description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.",
+              "title": "Forward Messages As Sent"
+            },
             "gateway_name": {
               "anyOf": [
                 {

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -6,9 +6,9 @@ from collections.abc import Mapping
 from types import MappingProxyType
 from typing import Final
 
-from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
 from fastapi.responses import JSONResponse
-from pydantic import JsonValue
+from pydantic import JsonValue, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
@@ -18,6 +18,7 @@ from litellm.anthropic_interface.exceptions import (
     AnthropicExceptionMapping,
 )
 from litellm.integrations.custom_guardrail import ModifyResponseException
+from litellm.litellm_core_utils.tool_fields import first_validation_error
 from litellm.llms.anthropic.pass_through.context_management import (
     AnthropicContextManagementError,
 )
@@ -32,6 +33,7 @@ from litellm.proxy.common_request_processing import (
     log_llm_api_exception,
     proxy_exception_from_http_exception,
     proxy_exception_from_route_error,
+    resolve_counted_tool_fields,
     resolve_litellm_call_id,
 )
 from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
@@ -47,6 +49,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
     with_litellm_call_id,
 )
 from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
+from litellm.router import Router
 
 router: Final = APIRouter()
 
@@ -91,6 +94,31 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO
     return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
 
 
+_TOKEN_COUNT_FIELDS: Final = ("model", "messages", "tools", "system")
+
+
+def _token_count_request(fields: Mapping[str, object]) -> TokenCountRequest | ProxyException:
+    try:
+        return TokenCountRequest.model_validate({name: fields[name] for name in _TOKEN_COUNT_FIELDS if name in fields})
+    except ValidationError as e:
+        return ProxyException(
+            message=first_validation_error(e),
+            type="invalid_request_error",
+            param=str(e.errors(include_url=False)[0]["loc"][0]),
+            code=status.HTTP_400_BAD_REQUEST,
+        )
+
+
+def _token_count_fields(
+    data: Mapping[str, object], auto_drop_params: bool, llm_router: Router | None, team_id: str | None
+) -> Mapping[str, object] | ProxyException:
+    fields: Final = {
+        name: data[name] for name in (*_TOKEN_COUNT_FIELDS, "drop_params", "additional_drop_params") if name in data
+    }
+    resolving: Final = {**fields, "drop_params": True} if auto_drop_params else fields
+    return resolve_counted_tool_fields(resolving, llm_router, team_id)
+
+
 def _strip_total_tokens_from_anthropic_response(response: object) -> None:
     """Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM
     injects into Anthropic /v1/messages responses.
@@ -351,7 +379,7 @@ async def count_tokens(
     
     Returns: {"input_tokens": <number>}
     """
-    from litellm.proxy.proxy_server import count_request_tokens, proxy_config
+    from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config
 
     litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
     try:
@@ -368,26 +396,21 @@ async def count_tokens(
         if not messages:
             raise HTTPException(status_code=400, detail="messages parameter is required")
 
-        # Create TokenCountRequest for the internal endpoint
-        from litellm.proxy._types import TokenCountRequest
-
-        token_request: Final = TokenCountRequest(
-            model=model_name,
-            messages=messages,
-            tools=data.get("tools"),
-            system=data.get("system"),
+        auto_drop_params: Final = should_auto_drop_params_for_agentic_cli(
+            request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
         )
+        token_fields: Final = _token_count_fields(data, auto_drop_params, llm_router, user_api_key_dict.team_id)
+        token_request: Final = (
+            token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields)
+        )
+        if isinstance(token_request, ProxyException):
+            return _anthropic_error_json_response(token_request, request)
 
         token_response: Final = await count_request_tokens(
             request=token_request,
             call_endpoint=True,
             messages_format="anthropic",
-            **_count_tokens_body_options(
-                data,
-                should_auto_drop_params_for_agentic_cli(
-                    request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
-                ),
-            ),
+            **_count_tokens_body_options(data, auto_drop_params),
         )
     except HTTPException as e:
         return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)

litellm/proxy/auth/auth_utils.py

@@ -551,6 +551,29 @@ def _iter_fallback_targets(value: object, depth: int) -> Iterator[str | Mapping[
                     yield from _iter_fallback_targets(target_list, depth + 1)
 
 
+def _reject_client_chosen_federation(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None:
+    reject_server_owned_wif_params(litellm_params)
+    if not manages_deployments:
+        reject_federated_credential_reference(litellm_params)
+
+
+def _iter_user_config_litellm_params(user_config: object) -> Iterator[Mapping[str, object]]:
+    config: Final = _coerce_metadata_to_dict(user_config)
+    if config is None:
+        return
+    if (defaults := _coerce_metadata_to_dict(config.get("default_litellm_params"))) is not None:
+        yield defaults
+    model_list: Final = config.get("model_list")
+    if not isinstance(model_list, list):
+        return
+    for deployment in model_list:
+        if (
+            isinstance(deployment, Mapping)
+            and (params := _coerce_metadata_to_dict(deployment.get("litellm_params"))) is not None
+        ):
+            yield params
+
+
 def iter_request_fallback_targets(request_body: Mapping[str, object]) -> Iterator[str | Mapping[str, object]]:
     for value in _iter_fallback_field_values(request_body):
         yield from _iter_fallback_targets(value, 0)
@@ -658,11 +681,11 @@ def is_request_body_safe(
                 _reject_url_valued_fallback_target(target_model)
         elif isinstance(target, str):
             _reject_url_valued_fallback_target(target)
+    for user_config_params in _iter_user_config_litellm_params(request_body.get("user_config")):
+        _reject_client_chosen_federation(user_config_params, manages_deployments=manages_deployments)
     litellm_params: Final = _coerce_metadata_to_dict(request_body.get("litellm_params"))
     if litellm_params is not None:
-        reject_server_owned_wif_params(litellm_params)
-        if not manages_deployments:
-            reject_federated_credential_reference(litellm_params)
+        _reject_client_chosen_federation(litellm_params, manages_deployments=manages_deployments)
         litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata"))
         if litellm_params_metadata is not None:
             _check_banned_params(

litellm/proxy/auth/model_checks.py

@@ -332,7 +332,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_
             else:
                 model = f"{wildcard_provider_prefix}/{model}"
         suffix_appended_wildcard_models.append(model)
-    return suffix_appended_wildcard_models or []
+    return list(dict.fromkeys(suffix_appended_wildcard_models))
 
 
 def expand_wildcard_deployments_for_model_info(

litellm/proxy/batches_endpoints/endpoints.py

@@ -17,6 +17,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, set_hidden_param
+from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
@@ -171,6 +172,8 @@ def _raise_not_found_when_openai_fallback_unservable(
         return
     if data.get("api_key") or litellm.api_key or litellm.openai_key or os.getenv("OPENAI_API_KEY"):
         return
+    if resolve_openai_workload_identity_config(api_key=None, api_base=None, litellm_params=data) is not None:
+        return
     raise ProxyException(
         message=not_found_message,
         type="invalid_request_error",

litellm/proxy/common_request_processing.py

@@ -80,6 +80,7 @@ from litellm.litellm_core_utils.served_output_texts import (
 from litellm.litellm_core_utils.streaming_handler import (
     backfill_missing_cache_usage_fields,
 )
+from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
 from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import (
     can_key_call_resolved_model,
@@ -129,6 +130,7 @@ from litellm.proxy.utils import (  # noqa: F401  # legacy module exports
 )
 from litellm.router import Router
 from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
+from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
 from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.router import RouterRateLimitError
@@ -633,6 +635,63 @@ def serialize_http_exception_detail(
     return str(detail), None
 
 
+_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
+_UNSEEN_DEPLOYMENT: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset())
+
+
+def _is_auto_router(params: Mapping[str, object]) -> bool:
+    model: Final = params.get("model")
+    return isinstance(model, str) and model.startswith(AUTO_ROUTER_MODEL_PREFIX)
+
+
+def _named_deployments(llm_router: Router, model: str, team_id: str | None) -> tuple[Mapping[str, object], ...]:
+    by_id: Final = None if model in llm_router.model_names else llm_router.get_deployment(model_id=model)
+    if by_id is not None:
+        return (_REQUEST_FIELDS.validate_python(by_id.litellm_params.model_dump(exclude_none=True)),)
+    group: Final = llm_router.get_model_from_alias(model=model) or model
+    if group not in llm_router.model_names or llm_router.get_model_list_from_routing_groups(model_name=group):
+        return ()
+    deployments: Final = tuple(
+        _REQUEST_FIELDS.validate_python(deployment["litellm_params"])
+        for deployment in llm_router.get_all_deployments(model_name=group, team_id=team_id)
+    )
+    return () if any(_is_auto_router(params) for params in deployments) else deployments
+
+
+def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
+    configured: Final[object] = getattr(llm_router, "default_litellm_params", None)
+    if configured is None:
+        return {}
+    return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None}
+
+
+def _counted_drop_policies(
+    data: Mapping[str, object], llm_router: Router | None, team_id: str | None
+) -> tuple[DropPolicy, ...]:
+    requested: Final = {**_router_defaults(llm_router), **data}
+    model: Final = data.get("model")
+    deployments: Final = (
+        _named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else ()
+    )
+    if not deployments:
+        return (drop_policy(requested), _UNSEEN_DEPLOYMENT)
+    return tuple(drop_policy({**deployment, **requested}) for deployment in deployments)
+
+
+def resolve_counted_tool_fields(
+    data: Mapping[str, object], llm_router: Router | None, team_id: str | None
+) -> Mapping[str, object] | ProxyException:
+    resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id))
+    if isinstance(resolved, MalformedField):
+        return ProxyException(
+            message=resolved.message,
+            type="invalid_request_error",
+            param=resolved.field,
+            code=status.HTTP_400_BAD_REQUEST,
+        )
+    return resolved
+
+
 def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
     raw_detail: Final = attribute_of(exc, "detail", str(exc))
     message, structured_fields = serialize_http_exception_detail(raw_detail)

litellm/proxy/db/db_transaction_queue/spend_update_queue.py

@@ -29,6 +29,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
     ) -> DBSpendUpdateTransactions:
         """Flush all updates from the queue and return all updates aggregated by entity type."""
         updates: Final = await self.flush_all_updates_from_in_memory_queue()
+        await self._emit_new_item_added_to_queue_event(queue_size=self.update_queue.qsize())
         if len(updates) > 0:
             verbose_proxy_logger.info(
                 "Spend tracking - flushed %d spend update items from in-memory queue",
@@ -48,6 +49,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
                 "Spend update queue is full. Aggregating all entries in queue to concatenate entries."
             )
             await self.aggregate_queue_updates()
+        await self._emit_new_item_added_to_queue_event(queue_size=self.update_queue.qsize())
 
     async def aggregate_queue_updates(self):
         """Concatenate all updates in the queue to reduce the size of in-memory queue"""
@@ -55,6 +57,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
         aggregated_updates: Final = self._get_aggregated_spend_update_queue_item(updates)
         for update in aggregated_updates:
             await self.update_queue.put(update)
+        await self._emit_new_item_added_to_queue_event(queue_size=self.update_queue.qsize())
 
     def _get_aggregated_spend_update_queue_item(
         self, updates: list[SpendUpdateQueueItem]

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py

@@ -32,6 +32,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}),
         unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
         fail_on_error=getattr(litellm_params, "fail_on_error", True),
+        forward_messages_as_sent=getattr(litellm_params, "forward_messages_as_sent", False),
         extra_headers=getattr(litellm_params, "extra_headers", None),
         guardrail_name=guardrail.get("guardrail_name", ""),
         event_hook=litellm_params.mode,

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py

@@ -7,10 +7,12 @@
 
 import fnmatch
 import os
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from typing import TYPE_CHECKING, Final, Literal, Optional
 
 import httpx
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from pydantic_core import PydanticSerializationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm._version import version as litellm_version
@@ -38,6 +40,10 @@ if TYPE_CHECKING:
     from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 GUARDRAIL_NAME: Final = "generic_guardrail_api"
+_CHAT_ROW: Final[TypeAdapter[AllMessageValues]] = TypeAdapter(AllMessageValues)
+_SENT_ROW: Final = TypeAdapter(Mapping[str, object])
+_SENT_ROWS: Final = TypeAdapter(Sequence[Mapping[str, object]])
+_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
 # Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*).
 _HEADER_VALUE_ALLOWLIST: Final = frozenset(
@@ -150,23 +156,65 @@ def _extract_inbound_headers(
     return None
 
 
+def _is_contained_in(part: JsonValue, whole: JsonValue) -> bool:
+    pending: Final[list[tuple[JsonValue, JsonValue]]] = [(part, whole)]  # mutable-ok: worklist, recursion is banned
+    while pending:
+        match pending.pop():
+            case dict() as part_dict, dict() as whole_dict:
+                if not part_dict.keys() <= whole_dict.keys():
+                    return False
+                pending.extend((value, whole_dict[key]) for key, value in part_dict.items())
+            case list() as part_list, list() as whole_list:
+                if len(part_list) != len(whole_list):
+                    return False
+                pending.extend(zip(part_list, whole_list, strict=True))
+            case part_leaf, whole_leaf if part_leaf != whole_leaf:
+                return False
+            case _:
+                pass
+    return True
+
+
+def _is_shown_row(shown: AllMessageValues, returned: AllMessageValues) -> bool:
+    return returned == shown
+
+
+def _is_echo(original: AllMessageValues, returned: AllMessageValues) -> bool:
+    """A server that parses rows with its own message types and echoes them back drops the
+    keys those types do not declare. That is still an echo as long as nothing changed and
+    every key the chat completion types declare is kept."""
+    if returned == original:
+        return True
+    try:
+        sent: Final[JsonValue] = _JSON.validate_json(_SENT_ROW.dump_json(original))
+        declared: Final[JsonValue] = _JSON.validate_json(
+            _CHAT_ROW.dump_json(_CHAT_ROW.validate_python(original), warnings="error")
+        )
+        echoed: Final[JsonValue] = _JSON.validate_python(returned)
+    except (ValidationError, PydanticSerializationError):
+        return False
+    return _is_contained_in(echoed, sent) and _is_contained_in(declared, echoed)
+
+
 def _structured_rows_to_write_back(
     original_rows: Sequence[AllMessageValues] | None,
-    shown_rows: Sequence[AllMessageValues] | None,
+    compared_rows: Sequence[AllMessageValues] | None,
     returned_rows: Sequence[AllMessageValues],
+    is_echo: Callable[[AllMessageValues, AllMessageValues], bool],
 ) -> tuple[AllMessageValues, ...] | None:
-    """The request model drops row keys its message types do not declare, so a
-    row the server echoes back verbatim is restored to the original row object.
-    A server that echoes every row back unchanged has not rewritten anything
-    per row, so its answer is read from texts, as it was before rows could be
-    returned at all."""
-    if original_rows is None or shown_rows is None or len(returned_rows) != len(original_rows):
+    """A row the server echoes back is restored to the original row object. A server
+    that echoes every row back has not rewritten anything per row, so its answer is
+    read from texts, as it was before rows could be returned at all."""
+    if original_rows is None or compared_rows is None or len(returned_rows) != len(original_rows):
         return tuple(returned_rows)
-    if all(returned == shown for shown, returned in zip(shown_rows, returned_rows)):
+    echoed: Final = tuple(
+        is_echo(compared, returned) for compared, returned in zip(compared_rows, returned_rows, strict=True)
+    )
+    if all(echoed):
         return None
     return tuple(
-        original if returned == shown else returned
-        for original, shown, returned in zip(original_rows, shown_rows, returned_rows)
+        original if was_echoed else returned
+        for original, returned, was_echoed in zip(original_rows, returned_rows, echoed, strict=True)
     )
 
 
@@ -200,6 +248,7 @@ class GenericGuardrailAPI(CustomGuardrail):
         additional_provider_specific_params: Mapping[str, object] | None = None,
         unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
         fail_on_error: bool | None = True,
+        forward_messages_as_sent: bool | None = False,
         extra_headers: list | None = None,
         streaming_end_of_stream_only: bool | None = None,
         streaming_sampling_rate: int | None = None,
@@ -235,6 +284,7 @@ class GenericGuardrailAPI(CustomGuardrail):
         self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback
 
         self.fail_on_error: bool = True if fail_on_error is None else fail_on_error
+        self.forward_messages_as_sent: bool = bool(forward_messages_as_sent)
 
         # Read by UnifiedLLMGuardrails.async_post_call_streaming_iterator_hook
         # via getattr(guardrail_to_apply, "streaming_*", default).
@@ -316,15 +366,17 @@ class GenericGuardrailAPI(CustomGuardrail):
         http_status_code: int | None = None,
     ) -> GenericGuardrailAPIInputs:
         status_suffix: Final = f" http_status_code={http_status_code}" if http_status_code else ""
+        error_type: Final = "HTTPStatusError" if isinstance(error, httpx.HTTPStatusError) else type(error).__name__
         verbose_proxy_logger.critical(
             "Generic Guardrail API error (fail-open). Proceeding without guardrail.%s "
-            "guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s",
+            "guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s error_type=%s",
             status_suffix,
             getattr(self, "guardrail_name", None),
             getattr(self, "api_base", None),
             input_type,
             getattr(logging_obj, "litellm_call_id", None) if logging_obj else None,
             getattr(logging_obj, "litellm_trace_id", None) if logging_obj else None,
+            error_type,
             exc_info=error,
         )
         # Keep flow going - treat as action=NONE (no modifications)
@@ -339,10 +391,23 @@ class GenericGuardrailAPI(CustomGuardrail):
             headers.update(self.headers)
         return headers
 
+    def _request_body(
+        self,
+        guardrail_request: GenericGuardrailAPIRequest,
+        structured_messages: Sequence[AllMessageValues] | None,
+    ) -> Mapping[str, object]:
+        body: Final[Mapping[str, object]] = guardrail_request.model_dump(mode="json")
+        if not self.forward_messages_as_sent or structured_messages is None:
+            return body
+        return {
+            **body,
+            "structured_messages": _SENT_ROWS.dump_python(_SENT_ROWS.validate_python(structured_messages), mode="json"),
+        }
+
     def _build_guardrail_return_inputs(
         self,
         *,
-        texts: list,
+        texts: list[str],
         images: list[str] | None,
         tools: list[ChatCompletionToolParam] | None,
         structured_messages: Sequence[AllMessageValues] | None,
@@ -361,8 +426,13 @@ class GenericGuardrailAPI(CustomGuardrail):
             return_inputs["tools"] = guardrail_response.tools
         elif tools:
             return_inputs["tools"] = tools
+        compared_rows, is_echo = (
+            (structured_messages, _is_echo) if self.forward_messages_as_sent else (shown_messages, _is_shown_row)
+        )
         rows_to_write_back: Final = (
-            _structured_rows_to_write_back(structured_messages, shown_messages, guardrail_response.structured_messages)
+            _structured_rows_to_write_back(
+                structured_messages, compared_rows, guardrail_response.structured_messages, is_echo
+            )
             if guardrail_response.structured_messages
             else None
         )
@@ -465,7 +535,7 @@ class GenericGuardrailAPI(CustomGuardrail):
                 litellm_version=litellm_version,
                 images=images,
                 tools=([GuardrailToolParam.model_validate(t) for t in tools] if tools else None),
-                structured_messages=structured_messages,
+                structured_messages=None if self.forward_messages_as_sent else structured_messages,
                 tool_calls=tool_calls,
                 additional_provider_specific_params=additional_params,
                 input_type=input_type,
@@ -478,7 +548,7 @@ class GenericGuardrailAPI(CustomGuardrail):
             # Use mode="json" to ensure all iterables are converted to lists
             response: Final = await self.async_handler.post(
                 url=self.api_base,
-                json=guardrail_request.model_dump(mode="json"),
+                json=dict(self._request_body(guardrail_request, structured_messages)),
                 headers=headers,
                 timeout=self.timeout,
             )

litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py

@@ -5,6 +5,7 @@ from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import (
     OpenAIModerationGuardrail,
 )
 from litellm.types.guardrails import SupportedGuardrailIntegrations
+from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
 
 if TYPE_CHECKING:
     from litellm.types.guardrails import Guardrail, LitellmParams
@@ -19,6 +20,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
 
     openai_moderation_guardrail: Final = OpenAIModerationGuardrail(
         guardrail_name=guardrail_name,
+        workload_identity_params=litellm_params.model_dump(include=set(OPENAI_WIF_KWARGS_KEYS), exclude_none=True),
         **{
             **litellm_params.model_dump(exclude_none=True),
             "api_key": litellm_params.api_key,

litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py

@@ -3,6 +3,8 @@
 OpenAI Moderation Guardrail Integration for LiteLLM
 """
 
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, Optional
 
 from fastapi import HTTPException
@@ -17,7 +19,14 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.workload_identity import (
+    OpenAIWorkloadIdentityConfig,
+    resolve_openai_bearer_token_async,
+    resolve_openai_workload_identity_config,
+)
 from litellm.proxy.common_utils.callback_utils import add_guardrail_scan_id
+from litellm.secret_managers.main import get_secret_str
 from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations
 from litellm.types.utils import (
     GenericGuardrailAPIInputs,
@@ -32,6 +41,21 @@ if TYPE_CHECKING:
     from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 
+def _config_secret(value: object) -> object:
+    return get_secret_str(value) if isinstance(value, str) and value.startswith("os.environ/") else value
+
+
+def _guardrail_workload_identity(
+    api_base: str, workload_identity_params: Mapping[str, object]
+) -> OpenAIWorkloadIdentityConfig | None:
+    try:
+        return resolve_openai_workload_identity_config(
+            api_key=None, api_base=api_base, litellm_params=workload_identity_params
+        )
+    except OpenAIError as incomplete_identity:
+        raise ValueError(f"OpenAI Moderation: {incomplete_identity.message}") from incomplete_identity
+
+
 class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
     """
     LiteLLM Built-in Guardrail for OpenAI Content Moderation.
@@ -55,6 +79,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
         model: Literal["omni-moderation-latest", "text-moderation-latest"] | None = None,
         streaming_end_of_stream_only: bool | None = None,
         streaming_sampling_rate: int | None = None,
+        workload_identity_params: Mapping[str, object] | None = None,
         **kwargs,
     ):
         """Initialize OpenAI Moderation guardrail handler."""
@@ -78,7 +103,11 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
         )
         self.streaming_sampling_rate: int = 5 if streaming_sampling_rate is None else streaming_sampling_rate
 
-        if not self.api_key:
+        self.workload_identity_params: Mapping[str, object] = MappingProxyType(
+            {key: _config_secret(value) for key, value in (workload_identity_params or {}).items()}
+        )
+
+        if not self.api_key and _guardrail_workload_identity(self.api_base, self.workload_identity_params) is None:
             raise ValueError(
                 "OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration."
             )
@@ -109,10 +138,13 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
 
         verbose_proxy_logger.debug("OpenAI Moderation guard request: %s", request_body)
 
+        bearer_token: Final = await resolve_openai_bearer_token_async(
+            static_api_key=self.api_key, api_base=self.api_base, litellm_params=self.workload_identity_params
+        )
         response: Final = await self.async_handler.post(
             url=f"{self.api_base}/moderations",
             headers={
-                "Authorization": f"Bearer {self.api_key}",
+                "Authorization": f"Bearer {bearer_token}",
                 "Content-Type": "application/json",
             },
             json=request_body,

litellm/proxy/guardrails/tool_name_extraction.py

@@ -6,23 +6,23 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl
 with standalone extractors for generate_content and MCP.
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
 from litellm.llms import load_guardrail_translation_mappings
+from litellm.llms.base_llm.guardrail_translation.utils import list_field
 from litellm.types.utils import CallTypes
 
 # Call types that have no guardrail translation handler; we use standalone extractors
 STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {}
 
 
-def _extract_generate_content_tool_names(data: dict) -> list[str]:
+def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]:
     """Google generateContent: tools[].functionDeclarations[].name"""
     names: Final[list[str]] = []
-    for tool in data.get("tools") or []:
-        if not isinstance(tool, dict):
-            continue
-        for decl in tool.get("functionDeclarations") or []:
+    for tool in list_field(data, "tools"):
+        for decl in list_field(tool, "functionDeclarations"):
             if isinstance(decl, dict) and decl.get("name"):
                 names.append(str(decl["name"]))
     return names

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -425,7 +425,7 @@ async def add_new_user_to_default_team(
     user_id: str,
     user_email: str | None,
     user_api_key_dict: UserAPIKeyAuth,
-    teams: list[str] | list[NewUserRequestTeam],
+    teams: Sequence[str | NewUserRequestTeam],
     prisma_client: "PrismaClient",
 ):
     tasks: Final[list[Awaitable[object]]] = []

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -28,7 +28,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError,
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
-from litellm.constants import LITELLM_PROXY_ADMIN_NAME
+from litellm.constants import LITELLM_PROXY_ADMIN_NAME, TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.ptu_pricing import (
@@ -1881,7 +1881,7 @@ async def delete_team_models(
     """
     deleted_model_ids: Final[list[str]] = []
     transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
-    async with transaction_client.tx() as tx:
+    async with transaction_client.tx(timeout=datetime.timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
         for team_id in team_ids:
             rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable)
             model_ids = [row.model_id for row in rows]

litellm/proxy/management_endpoints/team_endpoints.py

@@ -17,7 +17,7 @@ import traceback
 from collections.abc import Iterable, Mapping, Sequence
 from collections.abc import Set as AbstractSet
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from itertools import chain
 from types import MappingProxyType
 from typing import (
@@ -42,6 +42,7 @@ from litellm._internal_context import with_service_target
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.caching.redis_batch import active_request_redis_batches
+from litellm.constants import TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
 from litellm.integrations.prometheus import PrometheusLogger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy._types import (
@@ -4674,7 +4675,7 @@ async def delete_team(
     # or is still waiting on the lock, in which case its own re-read happens after this commits
     # and sees the row gone before it writes anything.
     delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}}
-    async with prisma_client.tx() as tx:
+    async with prisma_client.tx(timeout=timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
         for team_id in sorted(data.team_ids):
             await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
         await tx.litellm_teamtable.delete_many(where=delete_filter)

litellm/proxy/management_endpoints/ui_sso.py

@@ -89,6 +89,7 @@ from litellm.proxy._types import (
     Member,
     NewTeamRequest,
     NewUserRequest,
+    NewUserRequestTeam,
     NewUserResponse,
     ProxyErrorTypes,
     ProxyException,
@@ -118,7 +119,11 @@ from litellm.proxy.common_utils.html_forms.jwt_display_template import (
 from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form
 from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
-from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
+from litellm.proxy.management_endpoints.internal_user_endpoints import (
+    add_new_user_to_default_team,
+    check_if_default_team_set,
+    new_user,
+)
 from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO, LiteLLMGoogleSSO, create_generic_sso_provider
 from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import (
     id_jag_assertion_capture_gap,
@@ -2825,6 +2830,52 @@ async def insert_sso_user(
     return response
 
 
+DefaultTeamJoin: TypeAlias = Callable[
+    [LiteLLM_UserTable, Sequence[str | NewUserRequestTeam], PrismaClient], Awaitable[None]
+]
+
+
+async def _join_default_teams_as_admin(
+    user: LiteLLM_UserTable, teams: Sequence[str | NewUserRequestTeam], prisma_client: PrismaClient
+) -> None:
+    await add_new_user_to_default_team(
+        user_id=user.user_id,
+        user_email=user.user_email,
+        user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+        teams=teams,
+        prisma_client=prisma_client,
+    )
+
+
+def _default_teams_not_joined(user: LiteLLM_UserTable) -> tuple[str | NewUserRequestTeam, ...]:
+    joined: Final = frozenset(user.teams)
+    return tuple(
+        team
+        for team in check_if_default_team_set() or ()
+        if (team if isinstance(team, str) else team.team_id) not in joined
+    )
+
+
+async def _insert_sso_user_or_find_concurrent_insert(
+    insert: Awaitable[NewUserResponse],
+    user_id: str | None,
+    prisma_client: PrismaClient,
+    join_default_teams: DefaultTeamJoin = _join_default_teams_as_admin,
+) -> NewUserResponse | LiteLLM_UserTable:
+    try:
+        return await insert
+    except Exception:
+        users: Final = UserRepository(prisma_client, use_writer=True)
+        concurrent_insert: Final = None if user_id is None else await users.find_by_id(user_id)
+        if concurrent_insert is None:
+            raise
+        unjoined: Final = _default_teams_not_joined(concurrent_insert)
+        if not unjoined:
+            return concurrent_insert
+        await join_default_teams(concurrent_insert, unjoined, prisma_client)
+        return await users.find_by_id(concurrent_insert.user_id) or concurrent_insert
+
+
 @router.get(
     "/sso/get/ui_settings",
     tags=["experimental"],
@@ -3471,9 +3522,10 @@ class SSOAuthenticationHandler:
             else:
                 verbose_proxy_logger.info("user not in DB, inserting user into LiteLLM DB")
                 # user not in DB, insert User into LiteLLM DB
-                user_info = await insert_sso_user(
-                    result_openid=result,
-                    user_defined_values=user_defined_values,
+                user_info = await _insert_sso_user_or_find_concurrent_insert(
+                    insert=insert_sso_user(result_openid=result, user_defined_values=user_defined_values),
+                    user_id=None if user_defined_values is None else user_defined_values.get("user_id"),
+                    prisma_client=prisma_client,
                 )
             return user_info
         except Exception as e:

litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py

@@ -10,12 +10,14 @@ from pydantic import ConfigDict, TypeAdapter
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import ANTHROPIC_BATCHES_ROUTE
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_content_from_model_response,
 )
 from litellm.llms.anthropic import get_anthropic_config
+from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception
 from litellm.llms.anthropic.pass_through.stream_assembly import (
     build_complete_streaming_response,
     build_complete_streaming_response_legacy,
@@ -254,6 +256,26 @@ class AnthropicPassthroughLoggingHandler:
             ),
         )
 
+    @staticmethod
+    async def log_error_frame_as_failure(
+        attempt_logging_obj: LiteLLMLoggingObj,
+        request_body: Mapping[str, object],
+        all_chunks: Sequence[str | bytes],
+        error_event: tuple[str, str, int],
+    ) -> None:
+        """A stream closed at the provider's `event: error` frame is one failed attempt of the request: it logs
+        through the attempt-scoped copy its caller took at the close, so the usage it consumed never reaches
+        the success log of the attempt the router opens in its place."""
+        error_type, message, status_code = error_event
+        await asyncify(AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure)(
+            litellm_logging_obj=attempt_logging_obj, request_body=request_body, all_chunks=all_chunks
+        )
+        await attempt_logging_obj.dispatch_failure_handlers(
+            anthropic_error_frame_exception(error_type, message, status_code, str(attempt_logging_obj.model)),
+            "",
+            prefer_async_handlers=True,
+        )
+
     @staticmethod
     def _cost_partial_stream_or_zero(
         partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj

litellm/proxy/pass_through_endpoints/streaming_handler.py

@@ -165,6 +165,7 @@ class PassThroughStreamingHandler:
             )
 
         logging_scheduled = False
+        stream_ended = False  # rebind-ok: flipped once the upstream ran to its end, so the finally can tell a close-at-frame from a completed stream
         model_name: Final = PassThroughStreamingHandler._extract_model_for_cost_injection(
             request_body=request_body,
             url_route=url_route,
@@ -224,6 +225,7 @@ class PassThroughStreamingHandler:
                         )
                 if pending:
                     yield pending
+            stream_ended = True  # rebind-ok: the finally reads it to spot a close-at-frame
             # Stream completed cleanly.  When the proxy armed deferred
             # dispatch (post-call guardrails active), park the logging
             # coroutine on logging_obj instead of enqueueing now, so
@@ -267,13 +269,46 @@ class PassThroughStreamingHandler:
             # a success would double-log the same request.
             if not logging_scheduled and raw_bytes and response.status_code < 400:
                 logging_scheduled = True
+                superseded_error_event: Final = (
+                    None
+                    if stream_ended
+                    else PassThroughStreamingHandler._error_frame_closed_at(endpoint_type, raw_bytes)
+                )
                 try:
-                    GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=_build_logging_coroutine())
+                    GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+                        async_coroutine=(
+                            _build_logging_coroutine()
+                            if superseded_error_event is None
+                            else AnthropicPassthroughLoggingHandler.log_error_frame_as_failure(
+                                attempt_logging_obj=litellm_logging_obj.attempt_scoped_copy(),
+                                request_body=resolved_request_body,
+                                all_chunks=tuple(raw_bytes),
+                                error_event=superseded_error_event,
+                            )
+                        )
+                    )
                 except Exception as e:
                     verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e)
                 else:
                     bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params)
 
+    @staticmethod
+    def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None:
+        """The provider `event: error` frame of a stream the router closed because a retry or fallback took the
+        request over. That attempt is logged as its own failure, on a copy taken now so the retry's rebinding of
+        the shared logging object never reaches it. A stream that ran to its end or that the client closed keeps
+        logging through the success route, so its usage is still billed."""
+        from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
+            closing_superseded_attempt,
+            parse_anthropic_error_event,
+        )
+
+        if endpoint_type != EndpointType.ANTHROPIC or not closing_superseded_attempt():
+            return None
+
+        complete_frames, _ = split_complete_sse_frames(b"".join(raw_bytes))
+        return parse_anthropic_error_event(complete_frames)
+
     @staticmethod
     async def route_streaming_logging_to_handler(
         litellm_logging_obj: LiteLLMLoggingObj,

litellm/proxy/response_api_endpoints/endpoints.py

@@ -34,6 +34,7 @@ from litellm.integrations.custom_guardrail import ModifyResponseException
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
 from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+from litellm.litellm_core_utils.tool_fields import first_validation_error
 from litellm.llms.openai.responses.guardrail_translation.handler import (
     OpenAIResponsesHandler,
     build_blocked_response,
@@ -45,7 +46,12 @@ from litellm.proxy.auth.user_api_key_auth import (
     user_api_key_auth,
     user_api_key_auth_websocket,
 )
-from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, cost_headers, create_response
+from litellm.proxy.common_request_processing import (
+    ProxyBaseLLMRequestProcessing,
+    cost_headers,
+    create_response,
+    resolve_counted_tool_fields,
+)
 from litellm.proxy.common_utils.http_parsing_utils import (  # noqa: F401  # legacy module exports
     _read_request_body,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
     _safe_set_request_parsed_body,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
@@ -58,6 +64,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
 )
 from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
 from litellm.proxy.utils import handle_exception_on_proxy
+from litellm.router import Router
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.openai import (
     REASONING_EFFORT,
@@ -70,7 +77,6 @@ from litellm.types.utils import TokenCountResponse
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.proxy.utils import ProxyLogging
-    from litellm.router import Router
 
 router: Final = APIRouter()
 _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None)
@@ -1377,6 +1383,15 @@ def _proxy_token_counter() -> _TokenCounter:
 _token_counter_dep: Final = Depends(_proxy_token_counter)
 
 
+def _proxy_llm_router() -> Router | None:
+    from litellm.proxy.proxy_server import llm_router
+
+    return llm_router
+
+
+_llm_router_dep: Final = Depends(_proxy_llm_router)
+
+
 def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
     body: Final[_ResponsesApiErrorBody] = {
         "error": {
@@ -1433,6 +1448,8 @@ def _responses_input_as_token_count_messages(
 async def responses_input_tokens(
     request: Request,
     token_counter: _TokenCounter = _token_counter_dep,
+    llm_router: Router | None = _llm_router_dep,
+    user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep,
 ):
     """
     Count the input tokens of a Responses API request without calling the model.
@@ -1465,6 +1482,13 @@ async def responses_input_tokens(
             code="missing_required_parameter",
         )
 
+    counted_fields: Final = {
+        name: data[name] for name in ("model", "tools", "drop_params", "additional_drop_params") if name in data
+    }
+    token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id)
+    if isinstance(token_fields, ProxyException):
+        return _responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None)
+
     try:
         payload: Final[_TokenCountPayload] = {
             "model": model_name,
@@ -1472,9 +1496,13 @@ async def responses_input_tokens(
                 input_value=input_value,
                 instructions=data.get("instructions"),
             ),
-            "tools": data.get("tools"),
+            "tools": token_fields.get("tools"),
         }
         token_request: Final = TokenCountRequest.model_validate(payload)
+    except ValidationError as e:
+        return _responses_invalid_request_response(
+            message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None
+        )
     except Exception as e:
         return _responses_invalid_request_response(
             message=f"Invalid request for token counting: {e}", param=None, code=None

litellm/proxy/utils.py

@@ -3493,6 +3493,7 @@ class ProxyLogging:
         Is triggered when self._is_proxy_only_error() returns True
         """
         litellm_logging_obj: Logging | None = request_data.get("litellm_logging_obj", None)
+        set_up_by_request: Final = litellm_logging_obj is not None
         if litellm_logging_obj is None:
             from litellm._uuid import uuid
 
@@ -3557,7 +3558,9 @@ class ProxyLogging:
                 input = request_data["input"]
                 litellm_logging_obj.model_call_details["input"] = input
                 body_shape_call_type = CallTypes.aembedding.value
-            resolved_call_type: Final = _call_type_for_route(route) or body_shape_call_type
+            resolved_call_type: Final = _call_type_for_route(route) or (
+                None if set_up_by_request else body_shape_call_type
+            )
             if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
                 litellm_logging_obj.call_type = resolved_call_type
                 litellm_logging_obj.model_call_details["call_type"] = resolved_call_type

litellm/realtime_api/main.py

@@ -43,6 +43,7 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime
 from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
 from ..llms.openai.live.handler import OpenAILiveSessions
 from ..llms.openai.realtime.handler import OpenAIRealtime
+from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
 from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
 from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
 from ..llms.vertex_ai.vertex_llm_base import VertexBase
@@ -130,6 +131,17 @@ def _get_realtime_http_provider_config(
     return provider_config, resolved_api_base.rstrip("/"), resolved_api_key
 
 
+async def _realtime_http_bearer_token(
+    custom_llm_provider: str, api_key: str, api_base: str, litellm_params: GenericLiteLLMParams
+) -> str:
+    if custom_llm_provider != LlmProviders.OPENAI.value:
+        return api_key
+    bearer_token: Final = await resolve_openai_bearer_token_async(
+        static_api_key=api_key or None, api_base=api_base, litellm_params=litellm_params.model_dump(exclude_none=True)
+    )
+    return bearer_token or ""
+
+
 @wrapper_client
 async def acreate_realtime_client_secret(
     model: str | None = None,
@@ -180,7 +192,9 @@ async def acreate_realtime_client_secret(
         request_data["session"] = _with_resolved_session_model(request_data["session"], model_name)
     return await base_llm_http_handler.async_realtime_client_secret_handler(
         api_base=resolved_api_base,
-        api_key=resolved_api_key,
+        api_key=await _realtime_http_bearer_token(
+            custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
+        ),
         request_data=request_data,
         logging_obj=litellm_logging_obj,
         timeout=timeout or request_timeout,
@@ -253,7 +267,9 @@ async def acreate_realtime_transcription_session(
         request_data["input_audio_transcription"]["model"] = model_name
     return await base_llm_http_handler.async_realtime_transcription_session_handler(
         api_base=resolved_api_base,
-        api_key=resolved_api_key,
+        api_key=await _realtime_http_bearer_token(
+            custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
+        ),
         request_data=request_data,
         logging_obj=litellm_logging_obj,
         timeout=timeout or request_timeout,
@@ -418,17 +434,21 @@ async def _alive_session(
     api_base: str | None,
     api_key: str | None,
     litellm_metadata: Mapping[str, object],
+    litellm_params: Mapping[str, object],
 ) -> None:
     rejection: Final = _live_session_rejection(custom_llm_provider, litellm_metadata)
     if rejection is not None:
         _raise_live_session_rejection(rejection)
+    resolved_api_base: Final = api_base or litellm.api_base or "https://api.openai.com/"
     await openai_live_sessions.async_live_session(
         model=model,
         websocket=websocket,
         logging_obj=litellm_logging_obj,
         session_start=session_start,
-        api_base=api_base or litellm.api_base or "https://api.openai.com/",
-        api_key=api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY"),
+        api_base=resolved_api_base,
+        api_key=await resolve_openai_bearer_token_async(
+            static_api_key=openai_static_api_key(api_key), api_base=resolved_api_base, litellm_params=litellm_params
+        ),
     )
 
 
@@ -497,6 +517,7 @@ async def _arealtime(
             api_base=dynamic_api_base or litellm_params.api_base,
             api_key=dynamic_api_key,
             litellm_metadata=_build_litellm_metadata(kwargs),
+            litellm_params=kwargs,
         )
         return
 
@@ -558,8 +579,9 @@ async def _arealtime(
         )
     elif _custom_llm_provider == "openai":
         api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/"
-        # set API KEY
-        api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+        api_key = await resolve_openai_bearer_token_async(
+            static_api_key=openai_static_api_key(dynamic_api_key), api_base=api_base, litellm_params=kwargs
+        )
 
         await openai_realtime.async_realtime(
             model=model,
@@ -774,9 +796,18 @@ async def realtime_health_check(
         str | None, api_version or resolved_params.get("api_version")
     )
     url: str | None = None
+    health_check_api_key: Final = (
+        await resolve_openai_bearer_token_async(
+            static_api_key=openai_static_api_key(resolved_api_key),
+            api_base=resolved_api_base,
+            litellm_params=resolved_params,
+        )
+        if custom_llm_provider == LlmProviders.OPENAI.value
+        else resolved_api_key
+    )
     auth_headers: Final = _realtime_health_check_auth_headers(
         custom_llm_provider=custom_llm_provider,
-        api_key=resolved_api_key,
+        api_key=health_check_api_key,
         model_params=resolved_params,
     )
     if custom_llm_provider == "azure":

litellm/router.py

@@ -123,6 +123,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
     mask_sensitive_structure,
 )
 from litellm.litellm_core_utils.token_counter import offload_token_count
+from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception
 from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
 from litellm.llms.base_llm.vector_store.transformation import (
@@ -501,7 +502,9 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
 _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
 _CallbackT = TypeVar("_CallbackT")
 
-_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
+_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset(
+    {"model", "api_base", "api_key", "api_version", "responses_api_version"}
+)
 _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
 _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
 _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600
@@ -538,6 +541,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
 
 
 _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
+_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
 _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@@ -574,6 +578,21 @@ def _deployment_that_just_failed(request_metadata: object) -> str | None:
     return model_id if isinstance(model_id, str) else None
 
 
+def _drops_malformed_request_tools(
+    deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
+) -> bool:
+    if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"):
+        return False
+    set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
+    return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
+
+
+def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None:
+    if not isinstance(request_tools, (list, tuple)):
+        return None
+    return [*deployment_tools, *request_tools] if deployment_tools or request_tools else None
+
+
 def _deployment_pick_attributes(model: str, request_kwargs: Mapping[str, object] | None) -> Mapping[str, str | int]:
     """Bounded attributes for one deployment pick; attempt is 1-based within the current model group."""
     kwargs: Final = request_kwargs or {}
@@ -4508,7 +4527,9 @@ class Router:
         return deployment_pydantic_obj
 
     @staticmethod
-    def _merge_tools_from_deployment(deployment: dict, kwargs: dict) -> None:
+    def _merge_tools_from_deployment(
+        deployment: dict, kwargs: dict, defaults: Mapping[str, object] = _NO_DEFAULT_PARAMS
+    ) -> None:
         """
         Merge tools from deployment litellm_params with request kwargs.
         When both have tools, concatenate them (deployment tools first, then request tools).
@@ -4519,11 +4540,12 @@ class Router:
             dep_params = dep_params_raw
         else:
             dep_params = dep_params_raw.model_dump(exclude_none=True)
-        dep_tools: Final = dep_params.get("tools") or []
-        req_tools: Final = kwargs.get("tools") or []
-        if dep_tools or req_tools:
-            merged: Final = list(dep_tools) + list(req_tools)
-            kwargs["tools"] = merged
+        if _drops_malformed_request_tools(dep_params, defaults, kwargs):
+            for name in TOOL_FIELDS:
+                kwargs.pop(name, None)
+        merged_tools: Final = _merged_tools(dep_params.get("tools") or [], kwargs.get("tools") or [])
+        if merged_tools is not None:
+            kwargs["tools"] = merged_tools
         if "tool_choice" not in kwargs and dep_params.get("tool_choice") is not None:
             kwargs["tool_choice"] = dep_params["tool_choice"]
 
@@ -4544,7 +4566,7 @@ class Router:
             deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
         ):
             kwargs.pop(key, None)
-        self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs)
+        self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params)
 
         model_info = deployment.get("model_info", {}).copy()
         deployment_litellm_model_name = deployment["litellm_params"]["model"]
@@ -6042,6 +6064,7 @@ class Router:
         """
         from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
             aclose_if_supported,
+            aclose_superseded_attempt,
             parse_anthropic_error_event,
             parse_anthropic_refusal_stop_details,
         )
@@ -6051,6 +6074,14 @@ class Router:
 
         source_iterator: Final = response
 
+        async def close_source() -> None:
+            with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
+                await aclose_if_supported(source_iterator)
+
+        async def close_superseded() -> None:
+            with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
+                await aclose_superseded_attempt(source_iterator)
+
         async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]:
             from litellm.exceptions import MidStreamFallbackError
 
@@ -6137,11 +6168,11 @@ class Router:
                     model,
                     initial_kwargs,
                     wrapper,
+                    close_superseded,
                 ):
                     yield item
             finally:
-                with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
-                    await aclose_if_supported(source_iterator)
+                await close_source()
 
         # Referenced by stream_with_fallbacks via closure - assigned here, before
         # the generator body ever runs, so the reference resolves fine despite
@@ -6157,6 +6188,7 @@ class Router:
         model: str,
         initial_kwargs: dict[str, Any],  # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it
         wrapper: "FallbackAwareAnthropicMessagesStream",
+        close_superseded: Callable[[], Awaitable[None]],
     ) -> AsyncGenerator[bytes, None]:
         """Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
         from litellm.exceptions import MidStreamFallbackError
@@ -6176,6 +6208,7 @@ class Router:
         )
         if fallback_error is None:
             raise stream_error
+        await close_superseded()
         outcome: Final = await self._astream_retry_same_group(
             fallback_error,
             initial_kwargs,
@@ -11035,6 +11068,7 @@ class Router:
                 persist_across_reloads=False,
                 warning_display_name=model,
                 custom_llm_provider=custom_llm_provider,
+                add_to_provider_model_sets=False,
             )
 
         ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes

litellm/router_utils/common_utils.py

@@ -309,6 +309,9 @@ def filter_team_based_models(
     ]
 
 
+_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"})
+
+
 def _deployment_supports_web_search(deployment: dict) -> bool:
     """
     Check if a deployment supports web search.
@@ -343,14 +346,13 @@ def filter_web_search_deployments(
     if isinstance(healthy_deployments, dict):
         return healthy_deployments
 
-    is_web_search_request = False
-    tools: Final = request_kwargs.get("tools") or []
-    for tool in tools:
-        # These are the two websearch tools for OpenAI / Azure.
-        if tool.get("type") == "web_search" or tool.get("type") == "web_search_preview":
-            is_web_search_request = True
-            break
-
+    tools: Final[object] = request_kwargs.get("tools")
+    is_web_search_request: Final = isinstance(tools, (list, tuple)) and any(
+        isinstance(tool, dict)
+        and isinstance(tool_type := tool.get("type"), str)
+        and tool_type in _WEB_SEARCH_TOOL_TYPES
+        for tool in tools
+    )
     if not is_web_search_request:
         return healthy_deployments
 

litellm/router_utils/cooldown_handlers.py

@@ -329,6 +329,12 @@ def _should_run_cooldown_logic(
         )
         return False
 
+    if isinstance(original_exception, litellm.MalformedRequestFieldError):
+        verbose_router_logger.debug(
+            "Should Not Run Cooldown Logic: litellm rejected the request before calling the deployment"
+        )
+        return False
+
     if request_kwargs is not None and _is_per_user_provider_request(request_kwargs):
         verbose_router_logger.debug(
             "Should Not Run Cooldown Logic: the failure came from one caller's per-user provider session"

litellm/systemone/main.py

@@ -1,4 +1,6 @@
+import asyncio
 from collections.abc import Mapping
+from functools import partial
 from typing import Final
 
 import httpx
@@ -75,21 +77,24 @@ async def asystemone(
     assets: DecisionsAssetEntries | None = None,
     **kwargs: object,  # kwargs-ok: litellm_params forwarded by @client
 ) -> DecisionsResponse:
-    call: Final = _prepare_systemone_call(
-        model=model,
-        state=state,
-        questions=questions,
-        images=images,
-        assets=assets,
-        safety_identifier=safety_identifier,
-        audio=audio,
-        videos=videos,
-        api_key=api_key,
-        api_base=api_base,
-        timeout=timeout,
-        custom_llm_provider=custom_llm_provider,
-        extra_headers=extra_headers,
-        kwargs=kwargs,
+    call: Final = await asyncio.to_thread(
+        partial(
+            _prepare_systemone_call,
+            model=model,
+            state=state,
+            questions=questions,
+            images=images,
+            assets=assets,
+            safety_identifier=safety_identifier,
+            audio=audio,
+            videos=videos,
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            custom_llm_provider=custom_llm_provider,
+            extra_headers=extra_headers,
+            kwargs=kwargs,
+        )
     )
     return _systemone_response(await asend(call))
 

litellm/types/guardrails.py

@@ -1195,6 +1195,16 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
         description="Maximum decisions calls in flight at once on the guardrail in each proxy worker. Implemented by guardrail='decision_model'.",
     )
 
+    forward_messages_as_sent: bool | None = Field(
+        default=False,
+        description=(
+            "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the "
+            "guardrail exactly as the client sent it, so rows can carry provider-native content blocks and "
+            "keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with "
+            "undeclared keys removed."
+        ),
+    )
+
     extra_headers: list[str] | None = Field(
         default=None,
         description=(

litellm/types/litellm_params.py

@@ -67,6 +67,7 @@ class ProviderConnection:
     api_key: str | None = None
     api_base: str | None = None
     api_version: str | None = None
+    responses_api_version: str | None = None
     region_name: str | None = None
     headers: Mapping[str, str] | None = None
     provider_specific_header: "ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None" = None

litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py

@@ -1,7 +1,7 @@
 from collections.abc import Mapping, Sequence
 from typing import Any, Final, Literal, cast  # noqa: TID251  # JSON chat rows have no typed constructor across roles
 
-from pydantic import ConfigDict, Field
+from pydantic import ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer
 from typing_extensions import TypedDict
 
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -17,12 +17,20 @@ class GuardrailToolParam(LiteLLMBaseModel):
     """A tool forwarded verbatim to the guardrail for inspection.
 
     Built-in tools (code_interpreter, file_search, ...) have no ``function`` block
-    and stash their config in tool-specific keys, so only ``type`` is required and
-    ``extra="allow"`` preserves the rest instead of stripping it.
+    and stash their config in tool-specific keys, and provider-native tools such as
+    Gemini ``function_declarations`` carry no ``type`` at all, so nothing is required
+    and ``extra="allow"`` preserves the rest instead of stripping it.
     """
 
     model_config = ConfigDict(extra="allow")
-    type: str
+    type: str | None = None
+
+    @model_serializer(mode="wrap")
+    def _without_absent_type(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]:
+        dumped: Final = cast("Mapping[str, object]", handler(self))  # cast-ok: handler dumps this model's fields
+        if self.type is not None:
+            return dumped
+        return {key: value for key, value in dumped.items() if key != "type"}
 
 
 class GenericGuardrailAPIMetadata(TypedDict, total=False):
@@ -62,6 +70,15 @@ class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel):
         ),
     )
 
+    forward_messages_as_sent: bool | None = Field(
+        default=False,
+        description=(
+            "When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can "
+            "carry provider-native content blocks and keys the OpenAI chat types do not declare. False "
+            "(default) sends OpenAI-format chat rows with undeclared keys removed."
+        ),
+    )
+
     streaming_end_of_stream_only: bool | None = Field(
         default=None,
         description=(

litellm/types/router.py

@@ -496,6 +496,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
         def __init__(self, /, **data: object) -> None: ...  # kwargs-ok: provider parameters vary by deployment
 
     custom_llm_provider: str | None = None
+    responses_api_version: str | None = None
     tpm: int | None = None
     rpm: int | None = None
     itpm: int | None = None
@@ -703,6 +704,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
     api_key: str | None
     api_base: str | None
     api_version: str | None
+    responses_api_version: ReadOnly[str | None]
     timeout: float | str | httpx.Timeout | SDKTimeout | None  # writable-ok: preserve timeout updates
     stream_timeout: float | str | None
     max_retries: int | None
@@ -1400,8 +1402,9 @@ def reject_server_owned_wif_params(body: Mapping[str, object]) -> None:
     This lives here rather than under ``litellm.proxy`` so the router can call it on a
     post-authentication merge without core importing from the proxy package.
     """
+    submitted_names: Final = frozenset(key.removesuffix("[]") for key in body)
     for param in _server_owned_wif_litellm_params:
-        if param in body:
+        if param in submitted_names:
             raise ValueError(
                 f"Rejected Request: {param} is a server-owned workload identity federation or OAuth token exchange "
                 "parameter "

litellm/types/utils.py

@@ -3900,6 +3900,7 @@ class MirroredPricingParams(LiteLLMBaseModel):
     output_cost_per_character: float | None = None
     cache_read_input_token_cost: float | None = None
     cache_creation_input_token_cost: float | None = None
+    cache_creation_input_token_cost_above_1hr: float | None = None
     tiered_pricing: list[dict[str, Any]] | None = None
 
 
@@ -3927,7 +3928,6 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
     input_cost_per_token_priority: float | None = None
     input_cost_per_token_balanced: float | None = None
     input_cost_per_token_ultrafast: float | None = None
-    cache_creation_input_token_cost_above_1hr: float | None = None
     cache_creation_input_token_cost_above_100k_tokens: float | None = None
     cache_creation_input_token_cost_above_1hr_above_100k_tokens: float | None = None
     cache_creation_input_token_cost_above_200k_tokens: float | None = None

litellm/utils.py

@@ -92,6 +92,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
 from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.litellm_core_utils.tokenizer import strip_special_tokens
+from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
 from litellm.rust_bridge import tokenizer as tokenizer_dispatch
 from litellm.rust_bridge.catalog import decision
 from litellm.rust_bridge.configuration import Decision
@@ -474,6 +475,7 @@ from .exceptions import (
     BudgetExceededError,
     ContentPolicyViolationError,
     ContextWindowExceededError,
+    MalformedRequestFieldError,
     ModelNotMappedError,
     NotFoundError,
     OpenAIError,
@@ -1495,7 +1497,23 @@ def check_coroutine(value) -> bool:
     return get_coroutine_checker().is_async_callable(value)
 
 
-async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str):
+def _deployment_provider(kwargs: Mapping[str, object]) -> str:
+    model: Final = kwargs.get("model")
+    custom_llm_provider: Final = kwargs.get("custom_llm_provider")
+    api_base: Final = kwargs.get("api_base")
+    if not isinstance(model, str):
+        return ""
+    try:
+        return litellm.get_llm_provider(
+            model=model,
+            custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+            api_base=api_base if isinstance(api_base, str) else None,
+        )[1]
+    except BadRequestError:
+        return ""
+
+
+async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) -> dict[str, Any]:
     """
     Allow modifying the request just before it's sent to the deployment.
 
@@ -1506,7 +1524,16 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str)
     except ValueError:
         typed_call_type = None  # unknown call type
 
-    modified_kwargs = kwargs.copy()
+    route: Final[object] = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
+    resolved: Final = resolve_deployment_tool_fields(kwargs, route)
+    if isinstance(resolved, MalformedField):
+        raise MalformedRequestFieldError(
+            message=resolved.message,
+            model=kwargs.get("model"),
+            llm_provider=_deployment_provider(kwargs),
+            body={"type": "invalid_request_error", "param": resolved.field},
+        )
+    modified_kwargs = dict(resolved)
 
     custom_logger_class: Final = _get_cached_custom_logger()
     for callback in litellm.callbacks:
@@ -3418,6 +3445,7 @@ def register_model(
     persist_across_reloads: bool = True,
     warning_display_name: str | None = None,
     custom_llm_provider: str | None = None,
+    add_to_provider_model_sets: bool = True,
 ):
     """
     Register new / Override existing models (and their pricing) to specific providers.
@@ -3446,6 +3474,10 @@ def register_model(
     ``custom_llm_provider`` scopes the built-in cost-map match to entries for
     that provider, so a deployment id that happens to equal another provider's
     catalog key stays its own provider-less entry instead of merging into it.
+
+    ``add_to_provider_model_sets`` adds the keys to the per-provider model sets
+    behind provider resolution and wildcard model listing. Pass False when the
+    keys are opaque ids rather than model names (e.g. router deployment ids).
     """
 
     loaded_model_cost = {}
@@ -3531,56 +3563,10 @@ def register_model(
         _invalidate_model_cost_lowercase_map()
 
         verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key)
-        # add new model names to provider lists
-        if value.get("litellm_provider") == "openai":
-            if key not in litellm.open_ai_chat_completion_models:
-                litellm.open_ai_chat_completion_models.add(key)
-        elif value.get("litellm_provider") == "text-completion-openai":
-            if key not in litellm.open_ai_text_completion_models:
-                litellm.open_ai_text_completion_models.add(key)
-        elif value.get("litellm_provider") == "cohere":
-            if key not in litellm.cohere_models:
-                litellm.cohere_models.add(key)
-        elif value.get("litellm_provider") == "anthropic":
-            if key not in litellm.anthropic_models:
-                litellm.anthropic_models.add(key)
-        elif value.get("litellm_provider") == "openrouter":
-            split_string = key.split("/", 1)
-            if split_string[-1] not in litellm.openrouter_models:
-                litellm.openrouter_models.add(split_string[-1])
-        elif value.get("litellm_provider") == "vercel_ai_gateway":
-            if key not in litellm.vercel_ai_gateway_models:
-                litellm.vercel_ai_gateway_models.add(key)
-        elif value.get("litellm_provider") == "edenai":
-            if key not in litellm.edenai_models:
-                litellm.edenai_models.add(key)
-        elif value.get("litellm_provider") == "vertex_ai-text-models":
-            if key not in litellm.vertex_text_models:
-                litellm.vertex_text_models.add(key)
-        elif value.get("litellm_provider") == "vertex_ai-code-text-models":
-            if key not in litellm.vertex_code_text_models:
-                litellm.vertex_code_text_models.add(key)
-        elif value.get("litellm_provider") == "vertex_ai-chat-models":
-            if key not in litellm.vertex_chat_models:
-                litellm.vertex_chat_models.add(key)
-        elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
-            if key not in litellm.vertex_code_chat_models:
-                litellm.vertex_code_chat_models.add(key)
-        elif value.get("litellm_provider") == "ai21":
-            if key not in litellm.ai21_models:
-                litellm.ai21_models.add(key)
-        elif value.get("litellm_provider") == "nlp_cloud":
-            if key not in litellm.nlp_cloud_models:
-                litellm.nlp_cloud_models.add(key)
-        elif value.get("litellm_provider") == "aleph_alpha":
-            if key not in litellm.aleph_alpha_models:
-                litellm.aleph_alpha_models.add(key)
-        elif value.get("litellm_provider") == "bedrock":
-            if key not in litellm.bedrock_models:
-                litellm.bedrock_models.add(key)
-        elif value.get("litellm_provider") == "novita":
-            if key not in litellm.novita_models:
-                litellm.novita_models.add(key)
+        if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
+            litellm.openrouter_models.add(_key_str.split("/", 1)[-1])
+    if add_to_provider_model_sets:
+        litellm.add_known_models(loaded_model_cost)
     return model_cost
 
 
@@ -9252,7 +9238,7 @@ class ProviderConfigManager:
 
             return XAIAudioTranscriptionConfig()
         elif litellm.LlmProviders.OPENAI == provider:
-            if "gpt-4o" in model:
+            if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model):
                 return litellm.OpenAIGPTAudioTranscriptionConfig()
             else:
                 return litellm.OpenAIWhisperAudioTranscriptionConfig()

litellm/vector_stores/vector_store_registry.py

@@ -1,6 +1,6 @@
 # litellm/proxy/vector_stores/vector_store_registry.py
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
 from itertools import chain
 from typing import (
@@ -11,7 +11,7 @@ from typing import (
     get_args,
 )
 
-from pydantic import ConfigDict, TypeAdapter
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import remove_items_at_indices
@@ -34,6 +34,26 @@ else:
     PrismaClient = Any
 
 _DB_ROW: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
+_TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
+_STORE_ID_ITEMS: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
+
+
+def _named_store_ids(store_ids: object) -> tuple[str, ...]:
+    if isinstance(store_ids, str):
+        return (store_ids,)
+    try:
+        items: Final = _STORE_ID_ITEMS.validate_python(store_ids)
+    except ValidationError:
+        return ()
+    return tuple(store_id for store_id in items if isinstance(store_id, str))
+
+
+def _tool_vector_store_ids(tool: object) -> tuple[str, ...]:
+    try:
+        return _named_store_ids(_TOOL.validate_python(tool).get("vector_store_ids"))
+    except ValidationError:
+        return ()
 
 
 class VectorStoreIndexRegistry:
@@ -131,7 +151,7 @@ class VectorStoreRegistry:
 
         return VectorStoreToolParams(**kwargs)
 
-    def get_vector_store_ids_to_run(self, non_default_params: dict, tools: list[dict] | None = None) -> list[str]:
+    def get_vector_store_ids_to_run(self, non_default_params: dict, tools: object = None) -> list[str]:
         """
         Returns the vector store ids to run
 
@@ -175,7 +195,7 @@ class VectorStoreRegistry:
         tools_to_remove: Final[list[int]] = []
 
         for i, tool in enumerate(tools):
-            tool_vector_store_ids = tool.get("vector_store_ids", [])
+            tool_vector_store_ids = _tool_vector_store_ids(tool)
             if not tool_vector_store_ids:
                 continue
 
@@ -395,14 +415,15 @@ class VectorStoreRegistry:
 
         return vector_stores_to_run
 
-    def _get_vector_store_ids_from_tool_calls(self, tools: list[dict] | None, vector_store_ids: list[str]) -> list[str]:
+    def _get_vector_store_ids_from_tool_calls(self, tools: object, vector_store_ids: list[str]) -> list[str]:
         """
         Returns the vector store ids from the tool calls
         """
-        return [
-            *vector_store_ids,
-            *chain.from_iterable(tool["vector_store_ids"] for tool in tools or () if "vector_store_ids" in tool),
-        ]
+        try:
+            validated_tools: Final = _ITEMS.validate_python(tools)
+        except ValidationError:
+            return vector_store_ids
+        return [*vector_store_ids, *chain.from_iterable(_tool_vector_store_ids(tool) for tool in validated_tools)]
 
     def load_vector_stores_from_config(self, vector_stores_config: list[dict]):
         """

pyproject.toml

@@ -290,7 +290,7 @@ ci = [
     "langchain-openai==1.1.14",
     "langgraph>=1.2.4,<1.3.0",
     "langgraph-prebuilt>=1.1.0,<1.3.0",
-    "claude-agent-sdk==0.1.44",
+    "claude-agent-sdk==0.2.165",
     "google-cloud-speech==2.40.0",
 ]
 healthcheck = [

tests/code_coverage_tests/router_code_coverage.py

@@ -112,6 +112,8 @@ ignored_function_names = [
     "_get_wildcard_deployments",  # Tested through the get_model_list_of_routed_group wildcard test in test_router.py
     "_is_fallback_hop",  # Tested through the order fallback hop tests in test_router_order_fallback.py
     "_deployment_that_just_failed",  # Tested through the same-boundary hop test in test_router_order_fallback.py
+    "_merged_tools",  # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
+    "_drops_malformed_request_tools",  # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
     "_collect_ptu_share_teams",  # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
     "_resolve_team_ptu_ceiling",  # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
     "_register_continuation_prefill_check",  # Tested through the continuation tests in test_router.py

tests/e2e/llm_translation/responses_helpers.py

@@ -8,10 +8,13 @@ AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
 AZURE_OPENAI_API_VERSION: Final = "v1"
 
 
-def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody:
+def azure_openai_params(
+    api_version: str = AZURE_OPENAI_API_VERSION, responses_api_version: str | None = None
+) -> LiteLLMParamsBody:
     return LiteLLMParamsBody(
         model=AZURE_OPENAI_BACKEND,
         api_base="os.environ/AZURE_API_BASE",
         api_key="os.environ/AZURE_API_KEY",
         api_version=api_version,
+        responses_api_version=responses_api_version,
     )

tests/e2e/llm_translation/test_image_generation_e2e.py

@@ -24,7 +24,10 @@ from sdk_clients import SdkClients
 pytestmark = pytest.mark.e2e
 
 OPENAI_IMAGE_BACKEND: Final = "openai/gpt-image-1-mini"
-BEDROCK_IMAGE_BACKEND: Final = "bedrock/amazon.nova-canvas-v1:0"
+BEDROCK_IMAGE_BACKEND: Final = "bedrock/stability.stable-image-core-v1:1"
+# Nova Canvas reached end of life on 2026-09-30. Stable Image Core is active on Bedrock only in us-west-2:
+# https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-stability-ai-stable-image-core-1-0.html (2026-10-16)
+BEDROCK_IMAGE_REGION: Final = "us-west-2"
 
 
 class _OptionalImageBody(BaseModel):
@@ -96,10 +99,10 @@ class TestImageGeneration:
                 model=BEDROCK_IMAGE_BACKEND,
                 aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
                 aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
-                aws_region_name="os.environ/AWS_REGION",
+                aws_region_name=BEDROCK_IMAGE_REGION,
             ),
         )
-        images = sdk.openai(key).images.generate(model=model, prompt="Draw a cute cat", n=1, size="1024x1024")
+        images = sdk.openai(key).images.generate(model=model, prompt="Draw a cute cat")
         _assert_image_returned(images)
 
     @pytest.mark.skip(reason="stage red: product gap, /v1/images/generations 500s (aimage_generation TypeError) on missing prompt instead of 400")

tests/e2e/llm_translation/test_responses_e2e.py

@@ -40,7 +40,7 @@ from provider_edge import LiveEdge, start_provider_edge
 from provider_edge_bedrock import ConverseBodyCapture, bedrock_signer
 from proxy_client import ProxyClient
 from pydantic import BaseModel, TypeAdapter
-from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params
+from responses_helpers import AZURE_OPENAI_API_VERSION, AZURE_OPENAI_BACKEND, azure_openai_params
 from sdk_clients import NO_PROXY_CACHE, SdkClients
 
 pytestmark = pytest.mark.e2e
@@ -748,6 +748,70 @@ class TestResponses:
             f"/responses over azure openai preview returned no output text: {response.output!r}"
         )
 
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.RESPONSES,
+            providers=(Provider.AZURE,),
+            models=(AZURE_OPENAI_BACKEND,),
+            mode=Mode.NONSTREAM,
+        )
+    )
+    def test_responses_azure_openai_dated_ga_chat_api_version_serves_responses(
+        self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
+    ) -> None:
+        model: Final = _register(
+            proxy,
+            resources,
+            azure_openai_params(api_version="2024-10-21"),
+            prefix="e2e-responses-azure-dated-chat",
+        )
+        client: Final = sdk.openai(resources.key())
+
+        response: Final = client.responses.create(
+            model=model,
+            input="reply with one word",
+            instructions=INSTRUCTIONS,
+            extra_body=NO_PROXY_CACHE,
+        )
+        assert response.output_text.strip(), (
+            f"/responses over an azure deployment pinned to a dated GA chat api_version returned no output text: "
+            f"{response.output!r}"
+        )
+
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.RESPONSES,
+            providers=(Provider.AZURE,),
+            models=(AZURE_OPENAI_BACKEND,),
+            mode=Mode.NONSTREAM,
+        )
+    )
+    def test_responses_azure_openai_responses_api_version_overrides_a_pre_responses_api_version(
+        self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
+    ) -> None:
+        # Azure rejects /openai/responses below api-version 2025-03-01-preview (probed live 2026-10-15),
+        # so this call only succeeds if the proxy sends responses_api_version instead of api_version.
+        model: Final = _register(
+            proxy,
+            resources,
+            azure_openai_params(api_version="2024-10-01-preview", responses_api_version=AZURE_OPENAI_API_VERSION),
+            prefix="e2e-responses-azure-override",
+        )
+        client: Final = sdk.openai(resources.key())
+
+        response: Final = client.responses.create(
+            model=model,
+            input="reply with one word",
+            instructions=INSTRUCTIONS,
+            extra_body=NO_PROXY_CACHE,
+        )
+        assert response.output_text.strip(), (
+            f"/responses over an azure deployment with responses_api_version set returned no output text: "
+            f"{response.output!r}"
+        )
+
     @meta(
         Subject(
             domain=Domain.LLM_TRANSLATION,

tests/e2e/mcp/test_mcp_tool_search_e2e.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import Final
 
 import pytest
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp
 from e2e_config import provider_edge_base, unique_marker
 from e2e_http import unwrap
@@ -17,6 +18,19 @@ MCP_TOOL_SEARCH_TOOL_NAME: Final = "mcp_tool_search"
 OVERSIZED_DESCRIPTION: Final = f"OVERSIZED {'weather ' * 12_000}"
 
 
+class _SearchHit(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+
+
+_SEARCH_HITS: Final = TypeAdapter(tuple[_SearchHit, ...])
+
+
+def _hits_tool(hits: tuple[_SearchHit, ...], tool_name: str) -> bool:
+    return any(hit.name.endswith(f"-{tool_name}") for hit in hits)
+
+
 class TestMcpToolSearch:
     @pytest.mark.covers("mcp.tool_search.api_key.oversized_description_omitted")
     def test_native_tool_search_omits_oversized_description(
@@ -82,5 +96,6 @@ class TestMcpToolSearch:
         )
 
         assert result.is_error is not True, result.all_text
-        assert normal_tool_name in result.all_text
-        assert oversized_tool_name not in result.all_text
+        hits: Final = _SEARCH_HITS.validate_json(result.all_text)
+        assert _hits_tool(hits, normal_tool_name), hits
+        assert not _hits_tool(hits, oversized_tool_name), hits

tests/e2e/models.py

@@ -1366,6 +1366,7 @@ class LiteLLMParamsBody(BaseModel):
     litellm_credential_name: str | None = None
     api_base: str | None = Field(default=None, repr=False)
     api_version: str | None = None
+    responses_api_version: str | None = None
     realtime_protocol: str | None = None
     allowed_openai_params: list[str] | None = None
     drop_params: bool | None = None

tests/e2e/ui/helpers/playground.ts

@@ -2,42 +2,57 @@ import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/t
 import { navigateToPage, dismissFeedbackPopup } from "./navigation";
 import { Page } from "../fixtures/pages";
 
-/** Controls for the Test Key / Playground page, shared with the router-fallback specs. */
-
-/**
- * The configuration panel is rendered twice, docked and overlay, with one visible at a time.
- * Every control is narrowed to the visible copy or it trips strict mode against its hidden twin.
- */
 export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first();
 
-/** The model combobox, addressed by the placeholder its search input shows before selection. */
-export const modelSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByPlaceholder("Select a Model"));
+export const composer = (page: PlaywrightPage): Locator =>
+  onlyVisible(page.getByRole("group").filter({ has: page.getByRole("textbox", { name: "Message" }) }));
+
+export const modelSelect = (page: PlaywrightPage): Locator =>
+  onlyVisible(page.getByRole("combobox", { name: "Model", exact: true }));
 
 export const sendButton = (page: PlaywrightPage): Locator =>
-  onlyVisible(page.getByRole("button", { name: "Send message" }));
+  composer(page).getByRole("button", { name: "Send message" });
 
-/** The Virtual Key Source dropdown, addressed by the accessible name on its trigger. */
-export const keySourceSelect = (page: PlaywrightPage): Locator =>
-  onlyVisible(page.getByLabel("Virtual Key Source"));
+export const keySourceSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByLabel("Send as"));
 
 export async function openPlayground(page: PlaywrightPage): Promise<void> {
   await navigateToPage(page, Page.LlmPlayground);
   await dismissFeedbackPopup(page);
-  await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({
-    timeout: 20_000,
-  });
+  await expect(modelSelect(page)).toBeVisible({ timeout: 20_000 });
+  const searchHint = page.getByRole("button", { name: "Dismiss search hint" });
+  if (await searchHint.isVisible()) {
+    await searchHint.click();
+  }
+}
+
+export async function openRequestSettings(page: PlaywrightPage): Promise<void> {
+  await composer(page).getByRole("button", { name: "Settings", exact: true }).click();
+  await expect(keySourceSelect(page)).toBeVisible({ timeout: 10_000 });
+}
+
+export async function enterVirtualKey(page: PlaywrightPage, key: string): Promise<void> {
+  await openRequestSettings(page);
+  await keySourceSelect(page).click();
+  await onlyVisible(page.getByRole("option", { name: "Virtual key", exact: true })).click({ timeout: 15_000 });
+  const keyInput = onlyVisible(page.getByLabel("Virtual key", { exact: true }));
+  await expect(keyInput).toBeVisible({ timeout: 10_000 });
+  await keyInput.fill(key);
+  await page.keyboard.press("Escape");
+}
+
+export async function searchModels(page: PlaywrightPage, query: string): Promise<void> {
+  await modelSelect(page).click();
+  await onlyVisible(page.getByPlaceholder("Search models")).fill(query);
 }
 
 export async function selectModel(page: PlaywrightPage, model: string): Promise<void> {
-  const select = modelSelect(page);
-  await select.click();
-  // Virtualized: options outside the rendered window are absent from the DOM, so search first.
-  await select.fill(model);
+  await searchModels(page, model);
   await onlyVisible(page.getByRole("option", { name: model, exact: true })).click({ timeout: 15_000 });
+  await expect(modelSelect(page)).toContainText(model);
 }
 
 export async function sendMessage(page: PlaywrightPage, message: string): Promise<void> {
-  const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
+  const input = composer(page).getByRole("textbox", { name: "Message" });
   await expect(input).toBeVisible({ timeout: 15_000 });
   await input.fill(message);
   await sendButton(page).click();