Merge remote-tracking branch 'origin/main' into litellm_fix_object_permission_partial_upsert

.github/scripts/assert_ci_coverage.py

@@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
 DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
 COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
 GLOB_CHARS = frozenset("*?")
+DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
 
 # Trees whose jobs are sharded with no catch-all bucket, so every child that holds
 # tests has to be named by some shard or it runs nowhere. A child listed here is
@@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
     return frozenset(ast.literal_eval(values))
 
 
+def _owns(workflow: str, relative_path: str) -> bool:
+    owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
+    return owned_root is not None and relative_path.startswith(owned_root)
+
+
+def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
+    return frozenset[str]().union(
+        *(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
+    )
+
+
 def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
     runner: Final = repo_root / "tests/integration/run.py"
     if not runner.exists():
@@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         )
         != 1
     )
-    gha_tokens: Final = _invoked_test_tokens(
-        scalar
-        for path in (repo_root / ".github/workflows").glob("*.y*ml")
-        for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
+    workflow_tokens: Final = MappingProxyType(
+        {
+            path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
+            for path in (repo_root / ".github/workflows").glob("*.y*ml")
+        }
+    )
+    gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
+    circle_owned: Final = (
+        frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
     )
     findings: Final = (
         tuple(
-            Finding(path, "integration contract is also selected by GitHub Actions")
-            for path in paths
+            Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
+            for path in sorted(circle_owned)
             if any(_token_covers(token, path) for token in gha_tokens)
         )
         + tuple(
@@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
             for path in sorted(github_files)
             if not (repo_root / path).is_file()
         )
+        + tuple(
+            Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
+            for path in sorted(github_files)
+            if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
+        )
     )
     browser_commands: Final = tuple(
         scalar.value
@@ -688,13 +710,9 @@ def main() -> int:
     scalars = _all_scalars()
 
     integration_paths, ownership_findings = _integration_ownership()
-    test_findings = (
-        _uncovered_tests(
-            allowlist,
-            _invoked_selections(scalars)
-            + (Selection(included=integration_paths, ignored=frozenset()),),
-        )
-        + ownership_findings
+    test_findings = _uncovered_tests(
+        allowlist,
+        _invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
     )
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
     stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@@ -711,13 +729,25 @@ def main() -> int:
             test_findings,
             "Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
         )
+    if ownership_findings:
+        browser_remedy = (
+            " CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
+            "select them and the default Playwright config must exclude them."
+            if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
+            else ""
+        )
+        _report(
+            "integration contracts with the wrong CI owner",
+            ownership_findings,
+            "Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
+        )
     if dockerfile_findings:
         _report(
             "Dockerfiles that no CI job builds",
             dockerfile_findings,
             "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
         )
-    if stale_findings or test_findings or dockerfile_findings:
+    if stale_findings or test_findings or ownership_findings or dockerfile_findings:
         return 1
 
     _write(

.github/workflows/test-rust.yml

@@ -16,6 +16,10 @@ on:
       - "litellm/proxy/guardrails/**"
       - "litellm/utils.py"
       - "litellm/ocr/**"
+      - "litellm/chat_completions/dispatch.py"
+      - "litellm/embeddings/dispatch.py"
+      - "litellm/messages/dispatch.py"
+      - "litellm/responses/dispatch.py"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
       - "Makefile"
@@ -46,6 +50,10 @@ on:
       - "litellm/proxy/guardrails/**"
       - "litellm/utils.py"
       - "litellm/ocr/**"
+      - "litellm/chat_completions/dispatch.py"
+      - "litellm/embeddings/dispatch.py"
+      - "litellm/messages/dispatch.py"
+      - "litellm/responses/dispatch.py"
       - "litellm/llms/base_llm/ocr/**"
       - "litellm/llms/custom_httpx/llm_http_handler.py"
       - "Makefile"

.github/workflows/test-unit.yml

@@ -696,7 +696,6 @@ jobs:
               tests/unit/responses/litellm_completion_transformation
               tests/unit/responses/test_additional_tools.py
               tests/unit/responses/test_custom_tool_call.py
-              tests/unit/responses/test_dispatch.py
               tests/unit/responses/test_metadata_codex_callback.py
               tests/unit/responses/test_no_duplicate_spend_logs.py
               tests/unit/responses/test_null_test_fix.py

AGENTS.md

@@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
 
 `tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
-End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
+End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
 
 When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
@@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
 
 Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
 
-If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
+Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
 
 Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
 

README.md

@@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Nace (`nace`)](https://console.nace.ai/docs/drex) |  |  |  |  |  |  |  |  |  |  |
 | [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
 | [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

litellm-rust/crates/gateway-inference/src/messages.rs

@@ -24,7 +24,7 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
     "x-openrouter-experimental-metadata",
 ];
 const ANTHROPIC_API_HEADER_PROVIDERS: &str =
-    "anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
+    "anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
 
 pub async fn create(
     State(gateway): State<Arc<Gateway>>,

litellm-rust/crates/gateway-inference/tests/messages.rs

@@ -16,7 +16,7 @@ use wiremock::{
 #[rstest]
 #[case::anthropic("anthropic/test-model", "/v1/messages", true)]
 #[case::azure("azure_ai/test-model", "/anthropic/v1/messages", false)]
-#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", false)]
+#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", true)]
 #[tokio::test]
 async fn messages_reaches_the_provider_and_preserves_json_or_sse(
     #[case] model: &str,

litellm/__init__.py

@@ -1177,7 +1177,9 @@ def _build_models_by_provider() -> dict:
         "edenai": edenai_models,
         "datarobot": datarobot_models,
         "vertex_ai": vertex_chat_models
+        | vertex_code_chat_models
         | vertex_text_models
+        | vertex_code_text_models
         | vertex_anthropic_models
         | vertex_vision_models
         | vertex_language_models
@@ -1185,7 +1187,7 @@ def _build_models_by_provider() -> dict:
         | vertex_minimax_models
         | vertex_moonshot_models
         | vertex_zai_models,
-        "ai21": ai21_models,
+        "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
         "ollama": ollama_models,
@@ -1416,6 +1418,7 @@ from .exceptions import (
     InvalidRequestError,
     BadRequestError,
     ImageFetchError,
+    MalformedRequestFieldError as MalformedRequestFieldError,
     VectorStoreSearchError,
     NotFoundError,
     PermissionDeniedError,
@@ -1691,6 +1694,9 @@ if TYPE_CHECKING:
     from .llms.perplexity.decisions.transformation import (
         PerplexityDecisionsConfig as PerplexityDecisionsConfig,
     )
+    from .llms.nace.decisions.transformation import (
+        NaceDecisionsConfig as NaceDecisionsConfig,
+    )
     from .llms.typesafe.decisions.transformation import (
         TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
     )

litellm/_internal_context.py

@@ -37,6 +37,16 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
 _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
 
 
+@contextmanager
+def internal_sub_call() -> Generator[None]:
+    """A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
+    token: Final = is_internal_call.set(True)
+    try:
+        yield
+    finally:
+        is_internal_call.reset(token)
+
+
 @contextmanager
 def post_response_phase() -> Generator[None]:
     """Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

litellm/_lazy_imports_registry.py

@@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
     "HostedVLLMRerankConfig",
     "PerplexityDecisionsConfig",
     "TypeSafeDecisionsConfig",
+    "NaceDecisionsConfig",
     "OpenRouterDecisionsConfig",
     "FireworksAIDecisionsConfig",
     "CloudflareDecisionsConfig",
@@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "HostedVLLMRerankConfig",
     ),
     "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
+    "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
     "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
     "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
     "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

litellm/assistants/main.py

@@ -129,6 +129,7 @@ def get_assistants(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_assistants=aget_assistants,
         )
@@ -312,6 +313,7 @@ def create_assistants(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             create_assistant_data=create_assistant_data,
             client=client,
             async_create_assistants=async_create_assistants,
@@ -461,6 +463,7 @@ def delete_assistant(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             assistant_id=assistant_id,
             client=client,
             async_delete_assistants=async_delete_assistants,
@@ -637,6 +640,7 @@ def create_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             acreate_thread=acreate_thread,
         )
@@ -783,6 +787,7 @@ def get_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_thread=aget_thread,
         )
@@ -960,6 +965,7 @@ def add_message(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             a_add_message=a_add_message,
         )
@@ -1111,6 +1117,7 @@ def get_messages(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_messages=aget_messages,
         )
@@ -1308,6 +1315,7 @@ def run_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             arun_thread=arun_thread,
             event_handler=event_handler,

litellm/batches/batch_utils.py

@@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
 )
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import ModelInfo, Usage
-from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
+from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
 from litellm.utils import token_counter
 
 
@@ -569,6 +569,7 @@ def extract_file_access_credentials(
             # A federated deployment holds no api_key, so without these the fetch that reads a
             # finished batch's output has nothing to authenticate with and its cost is never billed.
             *sorted(ANTHROPIC_WIF_KWARGS_KEYS),
+            *sorted(OPENAI_WIF_KWARGS_KEYS),
         )
         for key in credential_keys:
             if key in litellm_params:

litellm/batches/main.py

@@ -309,6 +309,7 @@ def create_batch(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
             organization=organization,
             timeout=timeout,
             max_retries=optional_params.max_retries,
+            litellm_params=litellm_params,
         )
     elif custom_llm_provider == "azure":
         api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -819,6 +821,7 @@ def list_batches(
                 organization=organization,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -1016,6 +1019,7 @@ def cancel_batch(
                 organization=organization,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

litellm/caching/caching_handler.py

@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
 from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
 
 import litellm
-from litellm._internal_context import post_response_phase
+from litellm._internal_context import is_internal_call, post_response_phase
 from litellm._logging import print_verbose, verbose_logger
 from litellm.caching import InMemoryCache
 from litellm.caching.caching import S3Cache, response_cache_phase
@@ -511,7 +511,9 @@ class LLMCachingHandler:
                         custom_llm_provider=custom_llm_provider,
                     )
 
-                    if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
+                    if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
+                        cached_result=cached_result
+                    ):
                         logging_obj.handle_sync_success_callbacks_for_async_calls(
                             result=cached_result,
                             start_time=start_time,
@@ -820,7 +822,12 @@ class LLMCachingHandler:
             start_time (datetime): The start time of the operation.
             end_time (datetime): The end time of the operation.
             cache_hit (bool): Whether it was a cache hit.
+
+        An internal sub-call's hit fires nothing, since the parent call that folds it in logs it.
         """
+        if is_internal_call.get():
+            return
+
         from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 
         GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
     return not _supports_audio_input(model, litellm_params)
 
 
-def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
+def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
     return MappingProxyType(
         {
             key: value
@@ -1455,7 +1455,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
-        for key, value in _provider_metadata(raw_response.model_extra).items():
+        for key, value in responses_provider_metadata(raw_response.model_extra).items():
             setattr(model_response, key, value)
 
         # Preserve hidden params from the ResponsesAPIResponse, especially the headers
@@ -2139,7 +2139,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 from litellm.responses.utils import ResponseAPILoggingUtils
 
                 usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
-            provider_metadata: Final = _provider_metadata(response_data)
+            provider_metadata: Final = responses_provider_metadata(response_data)
             served_service_tier: Final = response_data.get("service_tier")
             return ModelResponseStream(
                 choices=[

litellm/constants.py

@@ -391,7 +391,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
 
-BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
+BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
 BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
 BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@@ -1859,6 +1859,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
 PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
 PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
 PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
+TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)  # 1 minute
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
 RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))

litellm/containers/README.md

@@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
         self,
         headers: dict,
         api_key: Optional[str] = None,
+        litellm_params: Optional[GenericLiteLLMParams] = None,
     ) -> dict:
         """Azure uses api-key header instead of Bearer token."""
         import litellm

litellm/cost_calculator.py

@@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
 from litellm.llms.openai.cost_calculation import (
     video_output_cost_per_second,
 )
+from litellm.llms.openai.transcriptions.gpt_transformation import (
+    OpenAIGPTAudioTranscriptionConfig,
+)
 from litellm.llms.perplexity.cost_calculator import (
     cost_per_token as perplexity_cost_per_token,
 )
@@ -325,6 +328,15 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
+def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
+    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+        return False
+    transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
+        model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
+    )
+    return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
+
+
 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
 
 
@@ -1426,32 +1438,15 @@ def completion_cost(
     vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
 ) -> float:
     """
-    Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
+    Return the USD cost of a LiteLLM call.
 
-    Parameters:
-        completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
+    Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
+    `messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
+    `litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
+    is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
 
-        [OPTIONAL PARAMS]
-        model (str): Optional. The name of the language model used in the completion calls
-        prompt (str): Optional. The input prompt passed to the llm
-        completion (str): Optional. The output completion text from the llm
-        total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
-        custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
-        custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
-
-    Returns:
-        float: The cost in USD dollars for the completion based on the provided parameters.
-
-    Exceptions:
-        Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
-
-
-    Note:
-        - If completion_response is provided, the function extracts token information and the model name from it.
-        - If completion_response is not provided, the function calculates token counts based on the model and input text.
-        - The cost is calculated based on the model, prompt tokens, and completion tokens.
-        - For certain models containing "togethercomputer" in the name, prices are based on the model size.
-        - For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
+    Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
+    or add it to model_prices_and_context_window.json.
     """
     try:
         call_type = infer_call_type(call_type, completion_response) or "completion"
@@ -1739,6 +1734,12 @@ def completion_cost(
                     audio_transcription_file_duration = _hidden.get(
                         "audio_transcription_duration",
                         getattr(completion_response, "duration", 0.0),
+                    ) or (
+                        completion_response.usage.seconds
+                        if isinstance(completion_response, TranscriptionResponse)
+                        and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
+                        and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
+                        else 0.0
                     )
                 elif call_type in _RERANK_CALL_TYPES:
                     if completion_response is not None and isinstance(completion_response, RerankResponse):
@@ -2048,7 +2049,7 @@ def completion_cost(
                 )
                 if idx == len(potential_model_names) - 1:
                     raise e
-        raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
+        raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
     except Exception as e:
         raise e
 
@@ -2063,13 +2064,20 @@ def get_response_cost_from_hidden_params(
 
     additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
     if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
-        response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
-        if response_cost is None:
-            return None
-        return float(additional_headers["llm_provider-x-litellm-response-cost"])
+        return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
     return None
 
 
+def _billable_header_cost(value: object) -> float | None:
+    if not isinstance(value, (str, int, float)):
+        return None
+    try:
+        cost: Final = float(value)
+    except ValueError:
+        return None
+    return cost if math.isfinite(cost) and cost >= 0 else None
+
+
 def response_cost_calculator(
     response_object: ModelResponse
     | EmbeddingResponse

litellm/decisions/call.py

@@ -57,7 +57,7 @@ from litellm.types.decisions import (
     UnsupportedDecisionsRequest,
 )
 from litellm.types.utils import LlmProviders, ProviderSpecificHeader
-from litellm.utils import ProviderConfigManager
+from litellm.utils import ProviderConfigManager, get_model_cost_key
 
 SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
 OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@@ -458,7 +458,7 @@ def prepare_call(
         ) from error
 
     try:
-        resolved_api_base, resolved_api_key = provider_config.resolve_connection(
+        resolved_api_base, static_api_key = provider_config.resolve_connection(
             dynamic_api_base or api_base, dynamic_api_key or api_key
         )
     except ValueError as error:
@@ -469,6 +469,7 @@ def prepare_call(
             model=model,
             llm_provider=provider,
         )
+    resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
     if resolved_api_key is None and provider_config.api_key_required:
         raise litellm.AuthenticationError(
             message=f"Missing API key for Decisions provider '{provider}'",
@@ -666,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
         {
             "model": f"{call.custom_llm_provider}/{call.model}",
             "custom_llm_provider": call.custom_llm_provider,
-            "provider_response_model": f"{call.custom_llm_provider}/{call.model}",
+            "provider_response_model": (
+                get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
+                or f"{call.custom_llm_provider}/{call.model}"
+            ),
             **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
         }
     )

litellm/decisions/main.py

@@ -1,5 +1,7 @@
+import asyncio
 import warnings
 from collections.abc import Mapping
+from functools import partial
 from typing import Final
 
 import httpx
@@ -104,19 +106,22 @@ async def adecisions(
     videos: DecisionsVideoClips | None = None,
     **kwargs: object,
 ) -> OpenAIDecisionResponse | DecisionsResponse:
-    call: Final = _prepare_decisions_call(
-        model=model,
-        decision_input=input,
-        questions=questions,
-        safety_identifier=safety_identifier,
-        audio=audio,
-        videos=videos,
-        api_key=api_key,
-        api_base=api_base,
-        timeout=timeout,
-        custom_llm_provider=custom_llm_provider,
-        extra_headers=extra_headers,
-        kwargs=kwargs,
+    call: Final = await asyncio.to_thread(
+        partial(
+            _prepare_decisions_call,
+            model=model,
+            decision_input=input,
+            questions=questions,
+            safety_identifier=safety_identifier,
+            audio=audio,
+            videos=videos,
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            custom_llm_provider=custom_llm_provider,
+            extra_headers=extra_headers,
+            kwargs=kwargs,
+        )
     )
     return await asend(call)
 

litellm/exceptions.py

@@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
         )
 
 
+class MalformedRequestFieldError(BadRequestError):
+    pass
+
+
 class UnprocessableEntityError(openai.UnprocessableEntityError):
     def __init__(
         self,

litellm/files/main.py

@@ -259,6 +259,7 @@ def create_file(
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
                 create_file_data=_create_file_request,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -388,6 +389,7 @@ def file_retrieve(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -569,6 +571,7 @@ def file_delete(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -775,6 +778,7 @@ def file_list(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -972,6 +976,7 @@ def file_content(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -1093,6 +1098,7 @@ def file_content_streaming(
             organization=openai_creds.organization,
             chunk_size=chunk_size,
             client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
+            litellm_params=optional_params.model_dump(exclude_none=True),
         )
     elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
         if not _is_async:

litellm/fine_tuning/main.py

@@ -239,6 +239,7 @@ def create_fine_tuning_job(
                 client=kwargs.get(
                     "client", None
                 ),  # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":

litellm/images/main.py

@@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes
 # BFL handlers
 from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
 from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
+from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
 from litellm.main import (
     azure_chat_completions,
     base_llm_aiohttp_handler,
@@ -505,6 +506,7 @@ def image_generation(
                 aimg_generation=aimg_generation,
                 client=client,
                 headers=headers,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "bedrock":
             if model is None:
@@ -681,10 +683,13 @@ def image_variation(
     api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
 
     if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
-        if api_key is None:
-            raise ValueError("API key is required for OpenAI image variations")
         if api_base is None:
             raise ValueError("API base is required for OpenAI image variations")
+        if api_key is None and (
+            resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
+            is None
+        ):
+            raise ValueError("API key is required for OpenAI image variations")
 
         response = openai_image_variations.image_variations(
             model_response=model_response,

litellm/integrations/langfuse/langfuse.py

@@ -103,6 +103,43 @@ def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_
     )
 
 
+def langfuse_trace_metadata(metadata: object, proxy_server_request: object) -> Mapping[str, object]:
+    entries: Final = _object_mapping(metadata) or _NO_METADATA
+    request: Final = _object_mapping(proxy_server_request)
+    headers: Final = _object_mapping(request.get("headers")) if request is not None else None
+    if headers is None:
+        return entries
+    header_entries: Final = {
+        key.removeprefix("langfuse_"): value
+        for key, value in _widened_items(headers)
+        if isinstance(key, str) and key.startswith("langfuse_")
+    }
+    return MappingProxyType({**entries, **header_entries})
+
+
+def langfuse_trace_id(
+    metadata: Mapping[str, object],
+    standard_trace_id: object,
+    litellm_trace_id: object,
+    litellm_call_id: object,
+    proxy_server_request: object,
+) -> object:
+    existing_trace_id: Final = metadata.get("existing_trace_id")
+    if existing_trace_id is not None:
+        return existing_trace_id
+    metadata_trace_id: Final = metadata.get("trace_id")
+    trace_id: Final = (
+        metadata_trace_id
+        if metadata_trace_id is not None
+        else standard_trace_id
+        if standard_trace_id is not None
+        else litellm_trace_id or litellm_call_id
+    )
+    if _is_session_header_trace(trace_id, metadata.get("session_id"), proxy_server_request):
+        return litellm_call_id or trace_id
+    return trace_id
+
+
 class _UsageObject(Protocol):
     """Token-count surface the Langfuse logger reads off a response usage payload."""
 
@@ -680,33 +717,19 @@ class LangFuseLogger:
             # Add default langfuse tags
             tags = self.add_default_langfuse_tags(tags=tags, kwargs=kwargs, metadata=metadata)
 
+            call_trace_id: Final = langfuse_trace_id(
+                metadata=clean_metadata,
+                standard_trace_id=standard_logging_object.get("trace_id")
+                if standard_logging_object is not None
+                else None,
+                litellm_trace_id=kwargs.get("litellm_trace_id"),
+                litellm_call_id=litellm_call_id,
+                proxy_server_request=litellm_params.get("proxy_server_request"),
+            )
             session_id: Final = clean_metadata.pop("session_id", None)
             trace_name = cast(str | None, clean_metadata.pop("trace_name", None))
-            trace_id = clean_metadata.pop("trace_id", None)
-            # Use standard_logging_object.trace_id if available (when trace_id from metadata is None)
-            # This allows standard trace_id to be used when provided in standard_logging_object
-            if trace_id is None and standard_logging_object is not None:
-                trace_id = cast(str | None, standard_logging_object.get("trace_id"))
-            # Fallback to litellm_call_id if no trace_id found
-            if trace_id is None:
-                trace_id = kwargs.get("litellm_trace_id") or litellm_call_id
+            clean_metadata.pop("trace_id", None)
             existing_trace_id: Final = clean_metadata.pop("existing_trace_id", None)
-            # If existing_trace_id is provided, use it as the trace_id to return
-            # This allows continuing an existing trace while still returning the correct trace_id
-            if existing_trace_id is not None:
-                trace_id = existing_trace_id
-            call_trace_id: Final = (
-                litellm_call_id or trace_id
-                if existing_trace_id is None
-                and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request"))
-                else trace_id
-            )
-            if call_trace_id != trace_id:
-                verbose_logger.debug(
-                    "Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace",
-                    trace_id,
-                    call_trace_id,
-                )
             requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ()))
             update_trace_keys: Final = (
                 requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else ()

litellm/integrations/prometheus.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     PROXY_REJECTED_BEFORE_ROUTING_KEY,
 )
 from litellm.exceptions import (
+    MalformedRequestFieldError,
     validate_rate_limit_category,
     validate_rate_limit_type,
 )
@@ -3104,7 +3105,11 @@ class PrometheusLogger(CustomLogger):
 
             # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
             # into requested_model and leave deployment-scoped labels empty.
-            deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
+            deployment_selected: Final = (
+                bool(model_id)
+                and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
+                and not isinstance(exception, MalformedRequestFieldError)
+            )
             if deployment_selected:
                 label_litellm_model_name = litellm_model_name
                 label_model_id = model_id
@@ -3368,7 +3373,7 @@ class PrometheusLogger(CustomLogger):
                 label_context=label_context,
             )
 
-            if remaining_requests:
+            if isinstance(remaining_requests, int):
                 """
                 "model_group",
                 "api_provider",
@@ -3382,7 +3387,7 @@ class PrometheusLogger(CustomLogger):
                 )
                 self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests)
 
-            if remaining_tokens:
+            if isinstance(remaining_tokens, int):
                 _labels = prometheus_label_factory(
                     supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_remaining_tokens_metric"),
                     enum_values=enum_values,

litellm/litellm_core_utils/core_helpers.py

@@ -318,7 +318,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
             items.pop(index)
 
 
-def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
+def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
     """
     Helper to get litellm metadata for spend tracking
 
@@ -326,7 +326,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
     and user_api_key values are in 'metadata'.
     """
     potential_spend_tracking_metadata_substring: Final = "user_api_key"
-    for key, value in metadata.items():
+    for key, value in dict(metadata).items():
         if potential_spend_tracking_metadata_substring in key:
             litellm_metadata[key] = value
     return litellm_metadata

litellm/litellm_core_utils/get_litellm_params.py

@@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "github_copilot_auth_type",
             "github_copilot_user_session",
             "fireworks_forward_user_id",
+            "responses_api_version",
             PROVIDER_AFFINITY_HEADER_KWARG_KEY,
         }
     )

litellm/litellm_core_utils/get_model_cost_map.py

@@ -17,6 +17,7 @@ import random
 import sys
 import threading
 import time
+import urllib.parse
 from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass, replace
 from datetime import datetime, timezone
@@ -24,11 +25,11 @@ from importlib.resources import files
 from itertools import chain
 from pathlib import Path
 from types import MappingProxyType
-from typing import Final, Protocol
+from typing import Final, Protocol, cast  # noqa: TID251  # adapts boto3 and botocore's dynamic client responses
 
 import httpx
 from pydantic import ConfigDict, TypeAdapter
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm import verbose_logger
 from litellm.constants import (
@@ -266,6 +267,19 @@ class _SyncGetClient(Protocol):
     def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
 
 
+class _S3Body(Protocol):
+    def read(self) -> bytes: ...
+
+
+class _S3GetObjectResponse(TypedDict):
+    Body: ReadOnly[_S3Body]
+    ETag: NotRequired[ReadOnly[str]]
+
+
+class _S3Client(Protocol):
+    def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
+
+
 _FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
 
 
@@ -276,6 +290,20 @@ def _default_reload_client() -> _AsyncGetClient:
     return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
 
 
+def _default_s3_client(timeout: int) -> _S3Client:
+    import boto3
+    from botocore.config import Config
+
+    return cast(  # cast-ok: boto3's generic client factory does not expose its generated S3 response type
+        _S3Client,
+        boto3.client(  # pyright: ignore[reportUnknownMemberType]  # generic factory returns Unknown service clients
+            "s3",
+            region_name=os.getenv("AWS_REGION_NAME") or None,
+            config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
+        ),
+    )
+
+
 def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
     reason: Final = f"{type(error).__name__} fetching {url}: {error}"
     if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
@@ -283,7 +311,41 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
     return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
 
 
+def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
+    try:
+        parsed: Final[object] = json.loads(content)
+    except ValueError as e:
+        return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
+    if not isinstance(parsed, dict):
+        return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
+    model_cost_map: Final = cast(  # cast-ok: JSON object keys are strings by definition
+        dict[str, object], parsed
+    )
+    return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
+
+
+def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
+    from botocore.exceptions import BotoCoreError, ClientError
+
+    parts: Final = urllib.parse.urlsplit(url)
+    bucket: Final = parts.netloc
+    key: Final = parts.path.lstrip("/")
+    if not bucket or not key:
+        return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
+
+    try:
+        client: Final = _default_s3_client(timeout)
+        response: Final = client.get_object(Bucket=bucket, Key=key)
+        content: Final = response["Body"].read()
+    except (BotoCoreError, ClientError) as error:
+        return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
+    return _classify_fetch_content(content, url, response.get("ETag"))
+
+
 async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
+    parts: Final = urllib.parse.urlsplit(url)
+    if parts.scheme == "s3":
+        return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
     try:
         response: Final = await client.get(url, timeout=timeout)
     except (httpx.HTTPError, httpx.InvalidURL) as e:
@@ -292,6 +354,9 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
 
 
 def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
+    parts: Final = urllib.parse.urlsplit(url)
+    if parts.scheme == "s3":
+        return _attempt_fetch_s3(url, timeout)
     try:
         response: Final = client.get(url, timeout=timeout)
     except (httpx.HTTPError, httpx.InvalidURL) as e:
@@ -307,15 +372,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
         )
     if response.is_error:
         return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
-    try:
-        parsed: Final = response.json()
-    except ValueError as e:
-        return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
-    if not isinstance(parsed, dict):
-        return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
-    return ModelCostMapReloaded(
-        model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
-    )
+    return _classify_fetch_content(response.content, url, response.headers.get("etag"))
 
 
 def _next_retry_wait(

litellm/litellm_core_utils/hidden_params.py

@@ -77,6 +77,11 @@ def get_hidden_params(obj: object) -> MutableMapping[str, object] | None:
     return _as_hidden_params_mapping(hidden_params)
 
 
+def served_from_cache(obj: object) -> bool:
+    hidden_params: Final = get_hidden_params(obj)
+    return hidden_params is not None and hidden_params.get("cache_hit") is True
+
+
 def set_hidden_params(obj: object, hidden_params: dict[str, object] | HiddenParams) -> None:
     if isinstance(obj, dict):
         obj[_HIDDEN_PARAMS_ATTR] = hidden_params

litellm/litellm_core_utils/litellm_logging.py

@@ -79,7 +79,12 @@ from litellm.litellm_core_utils.core_helpers import (
 )
 from litellm.litellm_core_utils.error_normalization import normalize_error
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
-from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_param
+from litellm.litellm_core_utils.hidden_params import (
+    HIDDEN_PARAMS_ATTR,
+    get_hidden_params,
+    served_from_cache,
+    set_hidden_param,
+)
 from litellm.litellm_core_utils.internal_call_metadata import (
     MODEL_ACCESS_GROUP_METADATA_KEY,
     is_unbilled_non_inference_call,
@@ -2056,7 +2061,7 @@ class Logging(LiteLLMLoggingBaseClass):
     ) -> None:
         if response_cost is None and not calculation_failed:
             return
-        if self.model_call_details.get("cache_hit") is True:
+        if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
             self.model_call_details["zero_cost_diagnostic"] = None
             return
         try:
@@ -2295,6 +2300,24 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
 
+    def attempt_scoped_copy(self) -> "Logging":
+        """A copy with its own model_call_details, so logging one attempt of a request through it leaves
+        the state the request's other attempts share untouched."""
+        attempt: Final = copy.copy(self)
+        attempt.model_call_details = dict(self.model_call_details)
+        return attempt
+
+    def carry_failed_attempt_usage(self) -> None:
+        """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
+        own usage while the request's failure log still bills every attempt."""
+        usage: Final = self.model_call_details.pop("combined_usage_object", None)
+        response_cost: Final = self.model_call_details.pop("response_cost", None)
+        if isinstance(usage, Usage):
+            self.model_call_details["failed_attempts_usage"] = (
+                *self.model_call_details.get("failed_attempts_usage", ()),
+                (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
+            )
+
     def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
         """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
         self.model_call_details["continued_partial_usage"] = usage
@@ -2662,6 +2685,8 @@ class Logging(LiteLLMLoggingBaseClass):
                 result = self._handle_a2a_response_logging(result=result)
 
             logging_result: Final = self.normalize_logging_result(result=result)
+            if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
+                self.model_call_details["cache_hit"] = True
 
             if isinstance(result, Response) and isinstance(
                 logging_result, (ModelResponse, EmbeddingResponse, TextCompletionResponse)
@@ -4327,6 +4352,9 @@ class Logging(LiteLLMLoggingBaseClass):
         row is built from this response rather than re-parsed from the served body."""
         self.bridged_model_response = response
 
+    def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
+        self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
+
     def billable_result(self, result: object) -> object:
         """The response the cost calculator prices: a /v1/messages body served through the chat
         bridge is replaced by the ModelResponse its spend row is built from, anything else is
@@ -4427,6 +4455,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """
         from litellm.completion_extras.litellm_responses_transformation.transformation import (
             LiteLLMResponsesTransformationHandler,
+            responses_provider_metadata,
         )
 
         served_id: Final = _provider_response_id(result)
@@ -4451,7 +4480,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 "usage-only ModelResponse to keep the spend_logs row.",
                 str(e),
             )
-            model_response: Final = litellm.ModelResponse(id=served_id)
+            model_response: Final = litellm.ModelResponse(
+                id=served_id, **responses_provider_metadata(result.model_extra)
+            )
             model_response.model = self.model
             usage: Final = result.usage
             if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage):
@@ -6679,6 +6710,11 @@ def _get_status_fields(
     return StandardLoggingPayloadStatusFields(llm_api_status=llm_api_status, guardrail_status=guardrail_status)
 
 
+def _served_from_cache_at_no_charge(result: object) -> bool:
+    hidden_params: Final = get_hidden_params(result)
+    return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
+
+
 def _extract_response_obj_and_hidden_params(
     init_response_obj: object,
     original_exception: Exception | None,
@@ -6841,8 +6877,9 @@ def get_standard_logging_object_payload(
         )  # maintain backwards compatibility with old request body check
 
         saved_cache_cost: float = 0.0
-        if cache_hit is True:
+        if cache_hit is True or served_from_cache(init_response_obj):
             id = f"{id}_cache_hit{time.time()}"  # do not duplicate the request id
+        if cache_hit is True:
             saved_cache_cost = (
                 logging_obj.response_cost_calculator(
                     result=init_response_obj,

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking:
 
     @staticmethod
     def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
-        if "web_search_options" in kwargs:
-            web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
-            return WebSearchOptions(**web_search_options)
+        web_search_options: Final = kwargs.get("web_search_options")
+        if isinstance(web_search_options, dict):
+            return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
 
         tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
             kwargs=kwargs, tool_type="web_search_preview"
@@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking:
     _get_web_search_options = get_web_search_options
 
     @staticmethod
-    def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
-        if "tools" in kwargs:
-            return cast(list[object], kwargs.get("tools", []))
-        return None
+    def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
+        tools: Final = kwargs.get("tools")
+        return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
 
     @staticmethod
     def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,17 +1,20 @@
 import asyncio
 import json
 import traceback
-from collections.abc import Coroutine, Mapping, Sequence
+from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
 from enum import Enum, auto
-from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
 
 from pydantic import TypeAdapter
 from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import redact_internal_details_from_client_message, verbose_logger
+from litellm._uuid import uuid
 from litellm.constants import (
+    BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
     REALTIME_METERED_USAGE_KEY,
     REALTIME_SESSION_FAILURE_LOGGED_KEY,
     REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@@ -98,6 +101,16 @@ class ScopedWebSocket(Protocol):
     def scope(self) -> _ASGIScope: ...
 
 
+@runtime_checkable
+class _ReplayScopeWebSocket(Protocol):
+    @property
+    def scope(self) -> MutableMapping[str, object]: ...
+
+
+def client_websocket_scope(websocket: object) -> MutableMapping[str, object] | None:
+    return websocket.scope if isinstance(websocket, _ReplayScopeWebSocket) else None
+
+
 class _ClientWebSocket(ScopedWebSocket, Protocol):
     async def send_text(self, data: str) -> None: ...
     async def receive_text(self) -> str: ...
@@ -116,10 +129,87 @@ def _decode_json_object(payload: str) -> Mapping[str, object]:
     return json.loads(payload)
 
 
+_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
+    {"type": "server_vad", "create_response": False}
+)
+_EMPTY_SESSION: Final[Mapping[str, object]] = MappingProxyType({})
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
+_GUARDRAIL_EVENT_ID_PREFIX: Final = "litellm_guardrail_"
+
+
+def _json_object(value: object) -> Mapping[str, object] | None:
+    return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
+
+
+def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
+    return _json_object(container.get(key))
+
+
+def _guardrail_event_id() -> str:
+    return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
+
+
+def _error_event_id(event: Mapping[str, object]) -> str | None:
+    error: Final = _mapping_field(event, "error")
+    caused_by: Final = None if error is None else error.get("event_id")
+    return caused_by if isinstance(caused_by, str) else None
+
+
+_TurnDetectionGate: TypeAlias = Callable[[object], Mapping[str, object]]
+
+
+def _turn_detection_with_auto_response_off(turn_detection: object) -> Mapping[str, object]:
+    client_turn_detection: Final = _json_object(turn_detection)
+    if client_turn_detection is None:
+        return dict(_GUARDRAIL_TURN_DETECTION_GATE)
+    return {**client_turn_detection, "create_response": False}
+
+
+def _nested_turn_detection_part(session: Mapping[str, object], gate: _TurnDetectionGate) -> Mapping[str, object]:
+    audio: Final = _mapping_field(session, "audio")
+    if audio is None:
+        return {}
+    audio_input: Final = _mapping_field(audio, "input")
+    if audio_input is None or "turn_detection" not in audio_input:
+        return {}
+    gated_input: Final = {
+        **audio_input,
+        "turn_detection": gate(audio_input["turn_detection"]),
+    }
+    return {"audio": {**audio, "input": gated_input}}
+
+
+def _with_create_response_disabled(
+    session: Mapping[str, object], gate: _TurnDetectionGate
+) -> Mapping[str, object] | None:
+    """``session`` with every turn_detection the client sent unable to auto-respond, or None when it sent none."""
+    flat_part: Final = {"turn_detection": gate(session["turn_detection"])} if "turn_detection" in session else {}
+    nested_part: Final = _nested_turn_detection_part(session, gate)
+    if not flat_part and not nested_part:
+        return None
+    return {**session, **flat_part, **nested_part}
+
+
+def _gate_session_update(
+    session: object, gate: _TurnDetectionGate, inject_gate_when_absent: bool
+) -> Mapping[str, object] | None:
+    """The session to forward under a transcript guardrail, or None when the frame needs no change."""
+    client_session: Final = _json_object(session)
+    if client_session is None:
+        return None
+    gated: Final = _with_create_response_disabled(client_session, gate)
+    if gated is not None or not inject_gate_when_absent:
+        return gated
+    return {**client_session, "turn_detection": gate(None)}
+
+
 class RealtimeEventNormalizer(Protocol):
     def should_drop(self, event: object) -> bool: ...
     def normalize(self, event: dict) -> dict: ...
     def patch_outgoing_session(self, session: dict) -> dict: ...
+    def gate_turn_detection(self, turn_detection: object, /) -> Mapping[str, object]: ...
+    @property
+    def deletes_conversation_items(self) -> bool: ...
 
 
 class RealtimeUsageMeter(Protocol):
@@ -154,6 +244,7 @@ class RealTimeStreaming:
         logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
     ):
         self.websocket: _ClientWebSocket = websocket
+        self._replay_scope: Final = client_websocket_scope(websocket)
         self.backend_ws = backend_ws
         self.logging_obj = logging_obj
         self._logging_worker = logging_worker
@@ -197,6 +288,8 @@ class RealTimeStreaming:
         # Track whether we have already sent the guardrail turn-detection update
         # that disables provider auto-response for transcription guardrails.
         self._guardrail_turn_detection_update_sent: bool = False
+        self._session_gate_sent: bool = False
+        self._guardrail_cancel_event_ids: frozenset[str] = frozenset()
         # Deferred Gemini Live setup: Pipecat may stream audio before session.update.
         # Buffer client audio until the backend acknowledges setup (setupComplete).
         self._backend_setup_complete: bool = provider_config is None or provider_config.requires_session_configuration()
@@ -213,6 +306,12 @@ class RealTimeStreaming:
         self._is_transcription_session: bool = force_transcription_model is not None
         # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
         self._event_normalizer = event_normalizer
+        self._gate_turn_detection: Final[_TurnDetectionGate] = (
+            _turn_detection_with_auto_response_off if event_normalizer is None else event_normalizer.gate_turn_detection
+        )
+        self._deletes_conversation_items: Final = (
+            event_normalizer is None or event_normalizer.deletes_conversation_items
+        )
         self._usage_meter = usage_meter
 
     @property
@@ -734,20 +833,37 @@ class RealTimeStreaming:
         except (json.JSONDecodeError, TypeError):
             return
 
+    def _make_turn_detection_update(self, turn_detection: Mapping[str, object]) -> dict[str, object]:
+        session: Final[dict[str, object]] = (
+            {"turn_detection": turn_detection}
+            if self._backend_uses_beta_protocol
+            else {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
+        )
+        return {"type": "session.update", "session": session}
+
     def _make_disable_auto_response_message(self) -> str:
         """Return a session.update that disables VAD auto-response."""
-        turn_detection: Final[dict[str, str | bool]] = {
-            "type": "server_vad",
-            "create_response": False,
-        }
-        if self._backend_uses_beta_protocol:
-            session: dict[str, object] = {"turn_detection": turn_detection}
-        else:
-            session = {
-                "type": "realtime",
-                "audio": {"input": {"turn_detection": turn_detection}},
-            }
-        return json.dumps({"type": "session.update", "session": session})
+        return json.dumps(self._make_turn_detection_update(self._gate_turn_detection(None)))
+
+    async def _send_guardrail_cancel(self) -> None:
+        event_id: Final = _guardrail_event_id()
+        self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
+        await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
+
+    async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
+        await self._send_guardrail_cancel()
+        if item_id is not None and self._deletes_conversation_items:
+            await self._send_to_backend(json.dumps({"type": "conversation.item.delete", "item_id": item_id}))
+
+    def _is_guardrail_cancel_error(self, event: Mapping[str, object]) -> bool:
+        return _error_event_id(event) in self._guardrail_cancel_event_ids
+
+    async def _send_session_gate_once(self) -> None:
+        """Gate the backend's default turn_detection ahead of the first client session.update, which overrides it."""
+        if self._session_gate_sent or not self._should_disable_vad_auto_response():
+            return
+        self._session_gate_sent = True
+        await self._send_to_backend(self._make_disable_auto_response_message())
 
     async def _maybe_send_guardrail_turn_detection_update(self) -> None:
         """Disable provider auto-response once when transcription guardrails are enabled."""
@@ -921,7 +1037,7 @@ class RealTimeStreaming:
                 if pre_block_backend_message is not None:
                     await self._send_to_backend(pre_block_backend_message)
                 if not self._is_transcription_session:
-                    await self._send_to_backend(json.dumps({"type": "response.cancel"}))
+                    await self._withdraw_blocked_turn(item_id)
                 # Send the policy violation hint (shows as small gray status text in UI).
                 await self.websocket.send_text(
                     json.dumps(
@@ -1051,10 +1167,7 @@ class RealTimeStreaming:
                 self.store_message(event_str)
                 self._capture_transcription_usage(event)
                 await self._send_event_to_client(event, event_str)
-                blocked = await self.run_realtime_guardrails(
-                    cast(str, transcript),
-                    item_id=cast(str | None, event.get("item_id")),
-                )
+                blocked = await self.run_realtime_guardrails(cast(str, transcript))
                 if not blocked and self._should_disable_vad_auto_response():
                     await self._send_to_backend(json.dumps({"type": "response.create"}))
                 continue
@@ -1086,7 +1199,7 @@ class RealTimeStreaming:
         if event_type == "session.created" and self._should_disable_vad_auto_response():
             self.store_message(event_obj)
             await self.websocket.send_text(self._event_to_client_json(event_obj))
-            await self._send_to_backend(self._make_disable_auto_response_message())
+            await self._send_session_gate_once()
             return True
 
         if event_type == "conversation.item.input_audio_transcription.completed":
@@ -1140,6 +1253,10 @@ class RealTimeStreaming:
                 if self._usage_meter is not None:
                     self._usage_meter.observe_backend_event(event)
 
+                if event.get("type") == "error" and self._is_guardrail_cancel_error(event):
+                    verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
+                    continue
+
                 if await self._handle_raw_backend_message(event, raw_response):
                     continue
 
@@ -1346,6 +1463,13 @@ class RealTimeStreaming:
         return item
 
     async def _receive_client_message(self) -> str | None:
+        refused_opening_message: Final = (
+            None
+            if self._replay_scope is None
+            else self._replay_scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
+        )
+        if isinstance(refused_opening_message, str):
+            return refused_opening_message
         try:
             return await self.websocket.receive_text()
         except Exception as e:  # noqa: BLE001  # whatever the client socket raises, the client is gone
@@ -1472,19 +1596,21 @@ class RealTimeStreaming:
                         and self.session_configuration_request is None
                         and not self._guardrail_turn_detection_update_sent
                         and self._should_disable_vad_auto_response()
-                    ):
-                        session: Mapping[str, object] | None = msg_obj.setdefault("session", {})
-                        if isinstance(session, dict):
-                            existing_td = session.get("turn_detection")
-                            if not isinstance(existing_td, dict):
-                                existing_td = {}
-                            existing_td["create_response"] = False
-                            session["turn_detection"] = existing_td
-                            message = json.dumps(msg_obj)
-                            guardrail_turn_detection_injected = True
-                            verbose_logger.debug(
-                                "Injected turn_detection into first session.update for audio transcription guardrails"
+                        and (
+                            gated_first := _gate_session_update(
+                                client_event.get("session", _EMPTY_SESSION),
+                                self._gate_turn_detection,
+                                inject_gate_when_absent=True,
                             )
+                        )
+                        is not None
+                    ):
+                        msg_obj["session"] = gated_first
+                        message = json.dumps(msg_obj)
+                        guardrail_turn_detection_injected = True
+                        verbose_logger.debug(
+                            "Injected turn_detection into first session.update for audio transcription guardrails"
+                        )
 
                     ## GUARDRAIL: Force ``create_response`` to False in any
                     # client-provided ``turn_detection`` so a later
@@ -1492,52 +1618,25 @@ class RealTimeStreaming:
                     # and bypass the transcription guardrail after the
                     # initial disable. Covers both the flat beta key and the
                     # nested GA ``audio.input.turn_detection`` shape, since
-                    # the GA remap below also accepts either form. Skipped
-                    # when the injection block above already ran for this
-                    # message, to avoid redundant double-serialization.
+                    # the GA remap below also accepts either form. A later
+                    # update with no turn_detection is forwarded as is: OpenAI
+                    # keeps the earlier disable for an unset field, and xAI
+                    # falls back to its manual default, which never auto-responds.
                     if (
                         msg_type == "session.update"
                         and not guardrail_turn_detection_injected
                         and self._should_disable_vad_auto_response()
+                        and (
+                            gated_later := _gate_session_update(
+                                client_event.get("session"),
+                                self._gate_turn_detection,
+                                inject_gate_when_absent=False,
+                            )
+                        )
+                        is not None
                     ):
-                        session = client_event.get("session")
-                        if isinstance(session, dict):
-                            td_overridden = False
-                            flat_td = session.get("turn_detection")
-                            flat_td_present = flat_td is not None
-                            if flat_td_present:
-                                if not isinstance(flat_td, dict):
-                                    flat_td = {}
-                                if flat_td.get("create_response") is not False:
-                                    flat_td["create_response"] = False
-                                    session["turn_detection"] = flat_td
-                                    td_overridden = True
-                            nested_td_present = False
-                            audio = session.get("audio")
-                            if isinstance(audio, dict):
-                                audio_input = audio.get("input")
-                                if isinstance(audio_input, dict):
-                                    nested_td = audio_input.get("turn_detection")
-                                    if nested_td is not None:
-                                        nested_td_present = True
-                                        if not isinstance(nested_td, dict):
-                                            nested_td = {}
-                                        if nested_td.get("create_response") is not False:
-                                            nested_td["create_response"] = False
-                                            audio_input["turn_detection"] = nested_td
-                                            td_overridden = True
-                            # Symmetric with the first-update injection block:
-                            # if the client omitted turn_detection entirely on
-                            # a subsequent session.update, still inject the
-                            # ``create_response: False`` override so the
-                            # transcription guardrail cannot be re-enabled by
-                            # any downstream merge that drops the original
-                            # disable.
-                            if not flat_td_present and not nested_td_present:
-                                session["turn_detection"] = {"create_response": False}
-                                td_overridden = True
-                            if td_overridden:
-                                message = json.dumps(msg_obj)
+                        msg_obj["session"] = gated_later
+                        message = json.dumps(msg_obj)
 
                     # GA compatibility: remap beta-style session fields only when
                     # the upstream is in GA mode. Beta upstreams expect the flat
@@ -1590,6 +1689,8 @@ class RealTimeStreaming:
                 # would permanently disable the injection if ``_send_to_backend``
                 # raised — neither this loop nor
                 # ``_maybe_send_guardrail_turn_detection_update`` would retry.
+                if msg_type == "session.update" and self.provider_config is None:
+                    await self._send_session_gate_once()
                 sent = await self._send_to_backend(message)
                 if guardrail_turn_detection_injected and sent:
                     self._guardrail_turn_detection_update_sent = True

litellm/litellm_core_utils/streaming_handler.py

@@ -1128,32 +1128,24 @@ class CustomStreamWrapper:
 
 
         """
-        if self.merge_reasoning_content_in_choices is True:
-            reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
-            if reasoning_content:
-                self._emitted_disqualifying_content = True
-                if self.sent_first_thinking_block is False:
-                    # Ensure content is not None before concatenation
-                    if model_response.choices[0].delta.content is None:
-                        model_response.choices[0].delta.content = ""
-                    model_response.choices[0].delta.content += "<think>" + reasoning_content
-                    self.sent_first_thinking_block = True
-                elif (
-                    self.sent_first_thinking_block is True
-                    and hasattr(model_response.choices[0].delta, "reasoning_content")
-                    and model_response.choices[0].delta.reasoning_content
-                ):
-                    model_response.choices[0].delta.content = reasoning_content
-            elif (
-                self.sent_first_thinking_block is True
-                and not self.sent_last_thinking_block
-                and model_response.choices[0].delta.content
-            ):
-                model_response.choices[0].delta.content = "</think>" + (model_response.choices[0].delta.content or "")
-                self.sent_last_thinking_block = True
+        if self.merge_reasoning_content_in_choices is not True:
+            return
+        delta: Final = model_response.choices[0].delta
+        reasoning_content: Final = getattr(delta, "reasoning_content", None)
+        content: Final = delta.content or ""
+        if reasoning_content:
+            self._emitted_disqualifying_content = True
+            closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
+            opening: Final = "" if self.sent_first_thinking_block else "<think>"
+            delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
+            self.sent_first_thinking_block = True
+            self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
+        elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
+            delta.content = "</think>" + content
+            self.sent_last_thinking_block = True
 
-            if hasattr(model_response.choices[0].delta, "reasoning_content"):
-                del model_response.choices[0].delta.reasoning_content
+        if hasattr(delta, "reasoning_content"):
+            del delta.reasoning_content
 
     def _dispatch_provider_chunk(
         self,

litellm/litellm_core_utils/tool_fields.py

@@ -0,0 +1,164 @@
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from functools import reduce
+from typing import Annotated, Final, TypeAlias
+
+from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
+from typing_extensions import ReadOnly, TypedDict
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+
+ROUTES_WITH_TOOLS: Final = frozenset(
+    {
+        "acompletion",
+        "aresponses",
+        "acompact_responses",
+        "anthropic_messages",
+        "agenerate_content",
+        "agenerate_content_stream",
+        "acreate_interaction",
+    }
+)
+TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
+_TOOLS: Final = TypeAdapter(list[object])
+_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
+_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
+
+
+def _schema_as_read(value: object) -> object:
+    if not value:
+        return None
+    if not isinstance(value, str):
+        return value
+    try:
+        return _JSON_OBJECT.validate_python(value)
+    except ValidationError:
+        return value
+
+
+_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
+
+
+class _ToolFunction(TypedDict, total=False):
+    parameters: ReadOnly[_Schema]
+
+
+class _ToolEntry(TypedDict, total=False):
+    function: ReadOnly[_ToolFunction | None]
+    parameters: ReadOnly[_Schema]
+    input_schema: ReadOnly[_Schema]
+    vector_store_ids: ReadOnly[list[str] | None]
+
+
+_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
+_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
+
+
+@dataclass(frozen=True, slots=True)
+class DropPolicy:
+    drop_params: bool
+    additional_drop_params: frozenset[str]
+
+    def drops(self, field: str) -> bool:
+        return self.drop_params or field in self.additional_drop_params
+
+
+@dataclass(frozen=True, slots=True)
+class MalformedField:
+    field: str
+    message: str
+    dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
+
+
+def _field_names(value: object) -> frozenset[str]:
+    try:
+        return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
+    except ValidationError:
+        return frozenset()
+
+
+def drop_policy(params: Mapping[str, object]) -> DropPolicy:
+    return DropPolicy(
+        drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
+        additional_drop_params=_field_names(params.get("additional_drop_params")),
+    )
+
+
+def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
+    error: Final = e.errors(include_url=False)[0]
+    return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
+
+
+def _tool_entry_error(index: int, entry: object) -> str | None:
+    try:
+        _TOOL_ENTRY.validate_python(entry, strict=True)
+    except ValidationError as e:
+        return first_validation_error(e, prefix=("tools", index))
+    return None
+
+
+def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
+    return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
+
+
+def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
+    tools: Final = data.get("tools")
+    if tools is None:
+        return None
+    try:
+        entries: Final = _TOOLS.validate_python(tools)
+    except ValidationError as e:
+        return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
+    entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
+    first_error: Final = next((error for error in entry_errors if error is not None), None)
+    if first_error is None:
+        return None
+    kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
+    return MalformedField(
+        "tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
+    )
+
+
+def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
+    return {name: value for name, value in data.items() if name != "web_search_options"}
+
+
+def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
+    try:
+        _WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
+    except ValidationError as e:
+        return MalformedField(
+            "web_search_options",
+            first_validation_error(e, prefix=("web_search_options",)),
+            _without_web_search_options,
+        )
+    return None
+
+
+def resolve_tool_fields(
+    data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
+) -> Mapping[str, object] | MalformedField:
+    checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
+    malformed: Final = tuple(field for field in checked if field is not None)
+    if not malformed:
+        return data
+    policies: Final = candidates()
+    rejected: Final = next(
+        (field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
+    )
+    if rejected is not None:
+        return rejected
+    return reduce(
+        lambda resolved, field: field.dropped(resolved),
+        (field for field in malformed if all(policy.drops(field.field) for policy in policies)),
+        data,
+    )
+
+
+def resolve_deployment_tool_fields(
+    kwargs: Mapping[str, object], route: object
+) -> Mapping[str, object] | MalformedField:
+    if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
+        return kwargs
+    return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
     effective_skip_tool_message_for_guardrail,
+    list_field,
     merge_guardrailed_scoped_messages,
     merge_returned_tools_into_request_tools,
     scoped_structured_message_indices,
@@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
             images=(),
         )
 
-    def extract_request_tool_names(self, data: dict) -> list[str]:
+    def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
         """Extract every tool name in an Anthropic messages request: tools[].name, plus
         tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
-        return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
+        return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
 
     @classmethod
     def _extract_input_text_and_images(

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -11,7 +11,7 @@ from litellm._logging import verbose_logger
 from litellm.exceptions import AuthenticationError
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
@@ -37,6 +37,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
         system: JsonValue = None,
         messages_format: TokenCountMessagesFormat = "anthropic",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using Anthropic's CountTokens API.

litellm/llms/anthropic/pass_through/adapters/handler.py

@@ -139,7 +139,7 @@ def request_safeguards_evaluator(
     )
 
 
-async def _prepare_context_managed_request(
+async def prepare_context_managed_request(
     *,
     model: str,
     messages: _AnthropicMessages,
@@ -152,6 +152,8 @@ async def _prepare_context_managed_request(
     user_api_key_auth: "UserAPIKeyAuth | None" = None,
     sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
     parent_trace_id: str | None = None,
+    count_only: bool = False,
+    drop_invalid_edits: bool = False,
 ) -> PolyfillResult | None:
     """Apply client compaction history, then optional context_management polyfill."""
     from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
@@ -193,6 +195,8 @@ async def _prepare_context_managed_request(
         user_api_key_auth=user_api_key_auth,
         sub_call_logging_kwargs=sub_call_logging_kwargs,
         parent_trace_id=parent_trace_id,
+        count_only=count_only,
+        drop_invalid_edits=drop_invalid_edits,
     )
 
     if polyfill_result is not None:
@@ -266,16 +270,14 @@ def _spec_has_non_compact_edits(
     return any(isinstance(edit.get("type"), str) and edit.get("type") != COMPACT_EDIT_TYPE for edit in edits)
 
 
-def _context_management_explicitly_dropped(additional_drop_params: list[str] | None) -> bool:
+def context_management_explicitly_dropped(additional_drop_params: object) -> bool:
     """True when the caller opted out of context_management via ``additional_drop_params``.
 
     ``drop_params`` deliberately does NOT gate the polyfill: ``context_management``
     is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and
     ``drop_params`` only exists to drop genuinely unsupported params.
     """
-    if not isinstance(additional_drop_params, list):
-        return False
-    return "context_management" in additional_drop_params
+    return isinstance(additional_drop_params, list) and "context_management" in additional_drop_params
 
 
 def _normalize_spec_edits(
@@ -291,7 +293,7 @@ def _normalize_spec_edits(
     if not context_management_spec:
         return None
 
-    if _context_management_explicitly_dropped(additional_drop_params):
+    if context_management_explicitly_dropped(additional_drop_params):
         return None
 
     from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
@@ -317,20 +319,23 @@ async def _run_polyfill_if_enabled(
     user_api_key_auth: "UserAPIKeyAuth | None" = None,
     sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
     parent_trace_id: str | None = None,
+    count_only: bool = False,
+    drop_invalid_edits: bool = False,
 ) -> PolyfillResult | None:
     """Run the async context_management polyfill if a spec is present.
 
     Returns ``None`` when the spec is empty or ``context_management`` is
     listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params``
-    does not disable the polyfill because context_management is a supported
+    does not disable a valid spec because context_management is a supported
     param). Raises ``AnthropicContextManagementError`` so the /v1/messages
-    endpoint can emit an Anthropic-format 400. All other exceptions are
+    endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits``
+    (``drop_params``) drops the invalid spec. All other exceptions are
     best-effort swallowed (matches v0 behavior).
     """
     if not context_management_spec:
         return None
 
-    if _context_management_explicitly_dropped(additional_drop_params):
+    if context_management_explicitly_dropped(additional_drop_params):
         return None
 
     try:
@@ -345,13 +350,17 @@ async def _run_polyfill_if_enabled(
             user_api_key_auth=user_api_key_auth,
             sub_call_logging_kwargs=sub_call_logging_kwargs,
             parent_trace_id=parent_trace_id,
+            count_only=count_only,
         )
     except AnthropicContextManagementError:
+        if drop_invalid_edits:
+            return None
         # Surface validation errors so the endpoint can emit an Anthropic-format
         # 400. Other exception types fall into the best-effort branch below.
         raise
     except Exception as e:
-        verbose_logger.exception("context_management polyfill: skipping edits due to error: %s", e)
+        log_failure: Final = verbose_logger.debug if count_only else verbose_logger.exception
+        log_failure("context_management polyfill: skipping edits due to error: %s", e)
         # Best-effort swallow is only safe for compact-only specs, where the
         # caller's compaction-block-slicing safety net produces a correct
         # (if degraded) result. When the spec also requested non-compact
@@ -709,6 +718,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         """Handle non-Anthropic models asynchronously using the adapter"""
         context_management: Final = kwargs.pop("context_management", None)
         additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
+        drop_invalid_edits: Final = litellm.drop_params is True or (
+            "drop_params" in kwargs and kwargs["drop_params"] is True
+        )
         litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None))
 
         proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
@@ -716,7 +728,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs)
         sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs)
 
-        polyfill_result: Final = await _prepare_context_managed_request(
+        polyfill_result: Final = await prepare_context_managed_request(
             model=model,
             messages=messages,
             tools=tools,
@@ -728,6 +740,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             user_api_key_auth=user_api_key_auth,
             sub_call_logging_kwargs=sub_call_logging_kwargs,
             parent_trace_id=sub_call_trace_id,
+            drop_invalid_edits=drop_invalid_edits,
         )
 
         effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages
@@ -841,6 +854,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         # bridge to it via ``run_async_function``.
         context_management: Final = kwargs.pop("context_management", None)
         additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
+        drop_invalid_edits: Final = litellm.drop_params is True or (
+            "drop_params" in kwargs and kwargs["drop_params"] is True
+        )
         # Deliberately do NOT auto-attach the proxy ``llm_router`` here:
         # ``run_async_function`` spawns a new event loop in a worker thread
         # to bridge to the async dispatcher, but the proxy router's httpx
@@ -866,7 +882,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
             request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
             polyfill_result = run_async_function(
-                _prepare_context_managed_request,
+                prepare_context_managed_request,
                 model=model,
                 messages=messages,
                 tools=tools,
@@ -878,6 +894,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 user_api_key_auth=user_api_key_auth,
                 sub_call_logging_kwargs=proxy_sub_call_logging_kwargs(request_kwargs),
                 parent_trace_id=proxy_sub_call_trace_id(request_kwargs),
+                drop_invalid_edits=drop_invalid_edits,
             )
 
         effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages

litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py

@@ -24,6 +24,7 @@ from typing_extensions import assert_never
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.hidden_params import set_hidden_params
 from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
 from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
@@ -39,6 +40,7 @@ from litellm.types.llms.anthropic import (
     UsageDelta,
     UsageIteration,
 )
+from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
 from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDeltaToolCall, Delta, Function
 
 if TYPE_CHECKING:
@@ -50,6 +52,7 @@ if TYPE_CHECKING:
 
 _STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType))
 _SSE_EVENT: Final = TypeAdapter(Mapping[str, object])
+_THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock)
 
 
 class _UsageDeltaWithIterations(UsageDelta, total=False):
@@ -106,6 +109,14 @@ def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
             assert_never(delta_type)
 
 
+def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None:
+    delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None
+    blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
+    first_block: Final = as_str_mapping(blocks[0]) if blocks else None
+    index: Final = first_block.get("index") if first_block is not None else None
+    return index if isinstance(index, int) else None
+
+
 def _optional_str_attr(obj: object, name: str) -> str | None:
     value: Final = _optional_attr(obj, name)
     return value if isinstance(value, str) else None
@@ -373,13 +384,44 @@ class _CombinedChunkSplitter:
             finish_delta.thinking_blocks = None
         return [content_chunk, finish_chunk]
 
+    @staticmethod
+    def _with_thinking_block(
+        chunk: "ModelResponseStream",
+        block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
+        *,
+        keep_role: bool,
+    ) -> "ModelResponseStream":
+        piece: Final = copy.deepcopy(chunk)
+        role: Final = getattr(piece.choices[0].delta, "role", None) if keep_role else None
+        thinking: Final = block.get("thinking") if block["type"] == "thinking" else None
+        piece.choices[0].delta = Delta(role=role, reasoning_content=thinking or None, thinking_blocks=[block])
+        return piece
+
+    @staticmethod
+    def _split_thinking_blocks(chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
+        """Anthropic streams one content block per thinking block, so a delta carrying several is split per block."""
+        choices: Final = _optional_attr_sequence(chunk, "choices")
+        if len(choices) != 1:
+            return (chunk,)
+        delta: Final = _optional_attr(choices[0], "delta")
+        blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
+        if len(blocks) < 2 or _optional_attr(delta, "content") or _optional_attr(delta, "tool_calls"):
+            return (chunk,)
+        return tuple(
+            _CombinedChunkSplitter._with_thinking_block(
+                chunk, _THINKING_BLOCK.validate_python(block), keep_role=position == 0
+            )
+            for position, block in enumerate(blocks)
+        )
+
     def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
         split_chunks: Final = tuple(
             split_chunk
             for combined_chunk in self._split(chunk)
             for split_chunk in self._split_by_payload_kind(combined_chunk)  # comprehension-ok: flattens nested splits
         )
-        return tuple(itertools.chain.from_iterable(self._route_tool_calls(split_chunk) for split_chunk in split_chunks))
+        block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
+        return tuple(itertools.chain.from_iterable(map(self._route_tool_calls, block_chunks)))
 
     def __iter__(self) -> "Iterator[ModelResponseStream]":
         return self
@@ -458,7 +500,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     sent_first_chunk: bool = False
     sent_content_block_start: bool = False
     sent_content_block_finish: bool = False
-    current_content_block_type: Literal["text", "tool_use", "thinking"] = "text"
+    current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text"
+    current_thinking_block_index: int | None = None
     sent_last_message: bool = False
     holding_chunk: ContentBlockDelta | None = None
     holding_stop_reason_chunk: MessageBlockDelta | None = None
@@ -1254,13 +1297,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         Also gates re-emission after a block transition: when an upstream
         chunk both *triggers* a new content block (its type differs from the
         active block) and *carries* delta content, that content belongs to
-        the new block. The synthesized ``content_block_start`` only ever
-        carries an empty body — see
-        ``_translate_streaming_openai_chunk_to_anthropic_content_block``,
-        which returns an empty ``TextBlock``/``ToolUseBlock``/thinking block —
-        so the trigger chunk's delta must be re-queued or the first token of
-        the new block (the first non-empty text/thinking delta, or bundled
-        tool arguments) is silently dropped.
+        the new block. The synthesized ``content_block_start`` from
+        ``_translate_streaming_openai_chunk_to_anthropic_content_block``
+        carries an empty text, tool-use or unsigned thinking body, and only a
+        signed thinking block's text rides on the start because its delta is
+        a ``signature_delta``. So the trigger chunk's delta must be re-queued
+        or the first token of the new block (the first non-empty text or
+        unsigned thinking delta, or bundled tool arguments) is silently
+        dropped.
 
         Delta types outside ``StreamingContentBlockDeltaType`` — the closed
         set the translate layer can produce — are treated as empty. The
@@ -1352,11 +1396,24 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                 original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name)
                 tool_block["name"] = original_name
 
+        thinking_block_index: Final = _first_thinking_block_index(chunk)
+        previous_thinking_block_index: Final = self.current_thinking_block_index
+        if thinking_block_index is not None or block_type != self.current_content_block_type:
+            self.current_thinking_block_index = thinking_block_index
+
         if block_type != self.current_content_block_type:
             self.current_content_block_type = block_type
             self.current_content_block_start = content_block_start
             return True
 
+        if (
+            thinking_block_index is not None
+            and previous_thinking_block_index is not None
+            and thinking_block_index != previous_thinking_block_index
+        ):
+            self.current_content_block_start = content_block_start
+            return True
+
         # For parallel tool calls, we'll necessarily have a new content block
         # if we get a function name since it signals a new tool call
         if block_type == "tool_use":

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -1627,7 +1627,7 @@ class LiteLLMAnthropicMessagesAdapter:
     def translate_streaming_openai_chunk_to_anthropic_content_block(
         self, choices: list[OpenAIStreamingChoice | StreamingChoices]
     ) -> tuple[
-        Literal["text", "tool_use", "thinking"],
+        Literal["text", "tool_use", "thinking", "redacted_thinking"],
         "ContentBlockContentBlockDict",
     ]:
         from litellm._uuid import uuid
@@ -1664,6 +1664,10 @@ class LiteLLMAnthropicMessagesAdapter:
                 thinking_blocks = choice.delta.thinking_blocks or []
                 if len(thinking_blocks) > 0:
                     thinking_block = thinking_blocks[0]
+                    if thinking_block["type"] == "redacted_thinking":
+                        return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
+                            type="redacted_thinking", data=thinking_block.get("data") or ""
+                        )
                     if thinking_block["type"] == "thinking":
                         thinking = thinking_block.get("thinking") or ""
                         signature = thinking_block.get("signature") or ""
@@ -1672,7 +1676,7 @@ class LiteLLMAnthropicMessagesAdapter:
                         assert isinstance(signature, str)
 
                         return "thinking", ChatCompletionThinkingBlock(
-                            type="thinking", thinking=thinking, signature=signature
+                            type="thinking", thinking=thinking if signature else "", signature=signature
                         )
             # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning
             # parsers) populate ``reasoning_content`` without ``thinking_blocks``.

litellm/llms/anthropic/pass_through/context_management/dispatcher.py

@@ -85,6 +85,7 @@ async def apply_context_management(
     user_api_key_auth: "UserAPIKeyAuth | None" = None,
     sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None,
     parent_trace_id: str | None = None,
+    count_only: bool = False,
 ) -> PolyfillResult:
     """Run edits in order; return a single ``PolyfillResult``.
 
@@ -128,6 +129,7 @@ async def apply_context_management(
                 user_api_key_auth=user_api_key_auth,
                 sub_call_logging_kwargs=sub_call_logging_kwargs,
                 parent_trace_id=parent_trace_id,
+                count_only=count_only,
             )
             if editor_is_async
             else await asyncify(editor)(

litellm/llms/anthropic/pass_through/context_management/editors/compact.py

@@ -642,6 +642,7 @@ async def apply_compact_20260112(
     user_api_key_auth: Optional["UserAPIKeyAuth"] = None,
     sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
     parent_trace_id: str | None = None,
+    count_only: bool = False,
 ) -> PolyfillResult:
     """Apply ``compact_20260112``; return a ``PolyfillResult``.
 
@@ -682,7 +683,7 @@ async def apply_compact_20260112(
 
     # Opt-in gate: no summary model configured → no-op (but still return the
     # Phase A-sliced/stripped messages so compaction blocks don't leak).
-    summary_model: Final = _read_summary_model_setting()
+    summary_model: Final = None if count_only else _read_summary_model_setting()
     if summary_model is None:
         applied["error"] = "summary_model_not_configured"
         # Slice-only forwarding: ``augmented_system`` already carries any prior

litellm/llms/anthropic/pass_through/messages/streaming_iterator.py

@@ -2,6 +2,7 @@ import asyncio
 import json
 import re
 from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
+from contextvars import ContextVar
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
 
@@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
         await stream.aclose()
 
 
+_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
+
+
+async def aclose_superseded_attempt(stream: object) -> None:
+    """Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
+    token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
+    try:
+        await aclose_if_supported(stream)
+    finally:
+        _CLOSING_SUPERSEDED_ATTEMPT.reset(token)
+
+
+def closing_superseded_attempt() -> bool:
+    return _CLOSING_SUPERSEDED_ATTEMPT.get()
+
+
 _RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
 
 

litellm/llms/anthropic/pass_through/responses_adapters/handler.py

@@ -43,6 +43,12 @@ _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
 _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
 
 
+def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
+    logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
+    if logging_obj is not None:
+        logging_obj.record_bridged_responses_api_response(result)
+
+
 def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
     """The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
     return extra_kwargs or {}
@@ -301,7 +307,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
         if not isinstance(result, ResponsesAPIResponse):
             raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
 
-        return await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
+        served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
+        _record_bridged_response(responses_kwargs, result)
+        return served
 
     @staticmethod
     def anthropic_messages_handler(

litellm/llms/azure/chat/gpt_5_transformation.py

@@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
                     status_code=400,
                     message=(
                         "Azure OpenAI does not support reasoning_effort='none' for this model. "
-                        "Supported values are: 'low', 'medium', and 'high'. "
                         "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
                         "`litellm_settings:\n drop_params: true`\n"
                         "Issue: https://github.com/BerriAI/litellm/issues/16704"

litellm/llms/azure/common_utils.py

@@ -2,6 +2,7 @@ import asyncio
 import hashlib
 import json
 import os
+import re
 from collections.abc import Callable, Mapping
 from functools import lru_cache
 from types import MappingProxyType
@@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import add_path_to_api_base
 
 azure_ad_cache: Final = DualCache()
+_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)")
 
 
 class _AzureAdTokenJson(TypedDict, total=False):
@@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM):
     ) -> dict[str, str]:  # mutable-ok: mirrors override contract
         return cls._base_validate_azure_environment(headers, litellm_params)
 
+    @staticmethod
+    def resolve_azure_api_base(api_base: str | None) -> str | None:
+        return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
+
     @staticmethod
     def _get_base_azure_url(
         api_base: str | None,
@@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM):
             default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
         """
 
-        api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
+        api_base = BaseAzureLLM.resolve_azure_api_base(api_base)
         if api_base is None:
             raise ValueError(
                 f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
@@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM):
             # ensure the request go to /openai/v1 and not just /openai
             if "/openai/v1" not in new_url:
                 parsed_url: Final = httpx.URL(new_url)
-                new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
+                new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1)))
 
         # Use the new query_params dictionary
         final_url: Final = httpx.URL(new_url).copy_with(params=query_params)

litellm/llms/azure/containers/transformation.py

@@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
         self,
         headers: dict,
         api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
         return BaseAzureLLM.base_validate_azure_environment(
             headers=headers,

litellm/llms/azure/fine_tuning/handler.py

@@ -1,4 +1,4 @@
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import Final, cast
 
 import httpx
@@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         self._ensure_training_type(create_fine_tuning_job_data)
 
@@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
         _is_async: bool = False,
         api_version: str | None = None,
-        litellm_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
         # Override to use Azure-specific client initialization
         if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI):
             client = None
 
         return self.get_azure_openai_client(
-            litellm_params=litellm_params or {},
+            litellm_params=dict(litellm_params) if litellm_params is not None else None,
             api_key=api_key,
             api_base=api_base,
             api_version=api_version,

litellm/llms/azure/responses/transformation.py

@@ -1,4 +1,7 @@
+import re
+from collections.abc import Mapping
 from copy import deepcopy
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
@@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem
 from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
+from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
@@ -25,6 +29,34 @@ else:
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
+_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+
+def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
+    value: Final = litellm_params.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None:
+    """
+    Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment
+    pinned to one for chat gets the default Responses version unless it sets responses_api_version.
+    """
+    api_version: Final = _str_param(litellm_params, "api_version")
+    if api_base is not None and "api-version" in httpx.URL(api_base).params:
+        return api_version
+    responses_api_version: Final = _str_param(litellm_params, "responses_api_version")
+    if responses_api_version is not None:
+        return responses_api_version
+    if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version):
+        return AZURE_DEFAULT_RESPONSES_API_VERSION
+    return api_version
+
+
+def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
+    return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
+
+
 class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     # Parameters not supported by Azure Responses API
     AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         - A complete URL string, e.g.,
         "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
         """
-        from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
-
+        resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
         return BaseAzureLLM.get_base_azure_url(
-            api_base=api_base,
-            litellm_params=litellm_params,
+            api_base=resolved_api_base,
+            litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
             route="/openai/responses",
             default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
         )

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -13,7 +13,7 @@ from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
     AzureAIAnthropicCountTokensHandler,
 )
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 azure_ai_anthropic_count_tokens_handler: Final = AzureAIAnthropicCountTokensHandler()
@@ -54,6 +54,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
         system: object | None = None,
         messages_format: TokenCountMessagesFormat = "anthropic",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using Azure AI Anthropic's CountTokens API.

litellm/llms/azure_ai/chat/transformation.py

@@ -1,6 +1,7 @@
 import copy
 import enum
 import re
+from collections.abc import AsyncIterator, Iterator
 from typing import TYPE_CHECKING, Final, cast
 
 import httpx
@@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import (
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
 from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
-from litellm.llms.openai.openai import OpenAIConfig
+from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
 from litellm.llms.xai.chat.transformation import XAIChatConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import ModelResponse, ProviderField
+from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
 from litellm.utils import add_path_to_api_base, supports_tool_choice
 
 if TYPE_CHECKING:
@@ -45,6 +46,14 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
 )
 
 
+class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
+    def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream:  # mutable-ok: mirrors override contract
+        usage: Final = chunk.get("usage")
+        if isinstance(usage, dict):
+            XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
+        return super().chunk_parser(chunk)
+
+
 class AzureAIGPT5Config(OpenAIGPT5Config):
     @classmethod
     def _model_map_lookup_name(cls, model: str) -> str:
@@ -319,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig):
         json_mode: bool | None = None,
     ) -> ModelResponse:
         model_response.model = f"azure_ai/{model}"
-        return super().transform_response(
+        response: Final = super().transform_response(
             model=model,
             raw_response=raw_response,
             model_response=model_response,
@@ -332,6 +341,20 @@ class AzureAIStudioConfig(OpenAIConfig):
             api_key=api_key,
             json_mode=json_mode,
         )
+        XAIChatConfig.fold_reasoning_tokens_into_completion(response)
+        return response
+
+    def get_model_response_iterator(
+        self,
+        streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
+        sync_stream: bool,
+        json_mode: bool | None = False,
+    ) -> "AzureAIChatCompletionResponseIterator":
+        return AzureAIChatCompletionResponseIterator(
+            streaming_response=streaming_response,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+        )
 
     def should_retry_llm_api_inside_llm_translation_on_http_error(
         self, e: httpx.HTTPStatusError, litellm_params: dict

litellm/llms/base_llm/base_model_iterator.py

@@ -6,16 +6,38 @@ import litellm
 
 if TYPE_CHECKING:
     import httpx
+from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
 from litellm.types.utils import (
     Choices,
     Delta,
     GenericStreamingChunk,
+    Message,
     ModelResponse,
     ModelResponseStream,
     StreamingChoices,
 )
 
 
+def _indexed_thinking_block(
+    block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, index: int
+) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
+    if block["type"] == "redacted_thinking":
+        return {**block, "index": index}
+    return {**block, "index": index}
+
+
+def _streaming_delta(message: Message) -> Delta:
+    thinking_blocks: Final = message.thinking_blocks if hasattr(message, "thinking_blocks") else None
+    return Delta(
+        **message.model_dump(exclude={"thinking_blocks"}),
+        thinking_blocks=(
+            None
+            if thinking_blocks is None
+            else [_indexed_thinking_block(block, index) for index, block in enumerate(thinking_blocks)]
+        ),
+    )
+
+
 def convert_model_response_to_streaming(
     model_response: ModelResponse,
 ) -> ModelResponseStream:
@@ -40,9 +62,7 @@ def convert_model_response_to_streaming(
             streaming_choices.append(
                 StreamingChoices(
                     index=choice.index,
-                    delta=Delta(
-                        **cast(Choices, choice).message.model_dump(),
-                    ),
+                    delta=_streaming_delta(cast(Choices, choice).message),
                     finish_reason=choice.finish_reason,
                 )
             )

litellm/llms/base_llm/base_utils.py

@@ -16,7 +16,13 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
-from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import (
+    CountContextManagement,
+    Message,
+    ProviderSpecificModelInfo,
+    TokenCountMessagesFormat,
+    TokenCountResponse,
+)
 
 
 class BaseTokenCounter(ABC):
@@ -32,6 +38,7 @@ class BaseTokenCounter(ABC):
         system: Any | None = None,
         messages_format: TokenCountMessagesFormat = "openai",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         pass
 

litellm/llms/base_llm/containers/transformation.py

@@ -81,6 +81,7 @@ class BaseContainerConfig(ABC):
         self,
         headers: dict,
         api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
         return {}
 

litellm/llms/base_llm/decisions/transformation.py

@@ -390,8 +390,24 @@ def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRe
     )
 
 
+_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
+
+
+def _normalize_answer(value: object) -> Mapping[str, object]:
+    answer: Final = _MAPPING.validate_python(value)
+    legend: Final = answer.get("legend")
+    if answer.get("type") != "score" or not isinstance(legend, list):
+        return answer
+    labels: Final = _LABELS.validate_python(legend)
+    return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
+
+
 def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
-    return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request)
+    response: Final = _MAPPING.validate_python(payload)
+    answers: Final = _MAPPING.validate_python(response.get("answers"))
+    normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
+    return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
 
 
 def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
@@ -495,6 +511,11 @@ class BaseDecisionsConfig(ABC):
     def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
         return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
 
+    def resolve_credential(
+        self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
+    ) -> str | None:
+        return api_key
+
     @staticmethod
     def _first_secret(names: tuple[str, ...]) -> str | None:
         return next((value for value in (get_secret_str(name) for name in names) if value), None)

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -4,7 +4,7 @@ import json
 from collections.abc import Callable, Iterator, Mapping, Sequence
 from typing import Final, TypeVar, cast  # noqa: TID251  # a rebuilt chat row has no typed constructor across roles
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
 from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
     return flat_name if isinstance(flat_name, str) else None
 
 
+_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
+_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
+
+
+def list_field(data: object, key: str) -> Sequence[object]:
+    try:
+        return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
+    except ValidationError:
+        return ()
+
+
 def anthropic_tool_names(tool: object) -> tuple[str, ...]:
     """Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
     ``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks

litellm/llms/base_llm/realtime/transcription_protocol.py

@@ -102,6 +102,16 @@ def json_integer(
     return value
 
 
+def uses_server_vad(
+    update: TranscriptionSessionUpdate, backend: str, drop_params: bool, error: ProtocolErrorType
+) -> bool:
+    if update.turn_detection_disabled:
+        return False
+    if update.turn_detection_type in (None, "server_vad") or drop_params:
+        return True
+    raise error(f"{backend} supports server_vad turn detection or null. Set drop_params to use server_vad instead")
+
+
 def new_event_id() -> str:
     return f"event_{uuid.uuid4().hex}"
 

litellm/llms/bedrock/chat/converse_handler.py

@@ -1,5 +1,5 @@
 import json
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
 from typing import Final
 
 import httpx
@@ -10,6 +10,7 @@ from litellm.anthropic_beta_headers_manager import (
 )
 from litellm.litellm_core_utils.get_litellm_params import stored_control_options
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+from litellm.llms.base_llm import base_model_iterator
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -21,7 +22,7 @@ from litellm.utils import CustomStreamWrapper
 
 from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, pop_aws_auth_params, run_aws_signing
 from ..common_utils import BedrockError, error_response_text, get_all_bedrock_regions
-from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
+from .invoke_handler import AWSEventStreamDecoder, make_call
 
 
 def make_sync_call(
@@ -36,7 +37,8 @@ def make_sync_call(
     fake_stream: bool = False,
     stream_chunk_size: int | None = None,
     timeout: float | httpx.Timeout | None = None,
-) -> tuple[MockResponseIterator | Iterator[object], httpx.Headers]:
+    optional_params: Mapping[str, object] | None = None,
+) -> tuple[base_model_iterator.MockResponseIterator | Iterator[object], httpx.Headers]:
     if client is None:
         client = get_httpx_client()  # Create a new client if none provided
 
@@ -64,14 +66,15 @@ def make_sync_call(
             model_response=litellm.ModelResponse(),
             stream=True,
             logging_obj=logging_obj,
-            optional_params={},
+            optional_params=dict(optional_params or {}),
             api_key="",
             data=data,
             messages=messages,
             encoding=litellm.encoding,
+            json_mode=json_mode,
         )
-        completion_stream: MockResponseIterator | Iterator[object] = MockResponseIterator(
-            model_response=model_response, json_mode=json_mode
+        completion_stream: base_model_iterator.MockResponseIterator | Iterator[object] = (
+            base_model_iterator.MockResponseIterator(model_response=model_response)
         )
     else:
         decoder: Final = AWSEventStreamDecoder(model=model, json_mode=json_mode)
@@ -158,6 +161,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             json_mode=json_mode,
             stream_chunk_size=stream_chunk_size,
             timeout=timeout,
+            optional_params=optional_params,
         )
         streaming_response: Final = CustomStreamWrapper(
             completion_stream=completion_stream,
@@ -290,7 +294,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream is True else None
         unencoded_model_id: Final = optional_params.pop("model_id", None)
         fake_stream = optional_params.pop("fake_stream", False)
-        json_mode: Final = optional_params.get("json_mode", False)
+        json_mode: Final = optional_params.get("json_mode") is True
         if unencoded_model_id is not None:
             modelId = self.encode_model_id(model_id=unencoded_model_id)
         else:
@@ -463,6 +467,7 @@ class BedrockConverseLLM(BaseAWSLLM):
                 fake_stream=fake_stream,
                 stream_chunk_size=stream_chunk_size,
                 timeout=timeout,
+                optional_params=optional_params,
             )
             streaming_response: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,

litellm/llms/bedrock/chat/invoke_handler.py

@@ -1,7 +1,7 @@
 import json
 import types
 from collections.abc import AsyncIterator, Iterator, Mapping
-from typing import TYPE_CHECKING, Final, cast
+from typing import TYPE_CHECKING, Final
 
 import httpx
 from pydantic import TypeAdapter, ValidationError
@@ -20,6 +20,7 @@ from litellm.llms.anthropic.chat.handler import (
     ModelResponseIterator as AnthropicModelResponseIterator,
 )
 from litellm.llms.anthropic.chat.transformation import JsonModeRequest
+from litellm.llms.base_llm import base_model_iterator
 from litellm.llms.base_llm.base_utils import ResponseFormatSchema
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -32,12 +33,8 @@ from litellm.types.llms.openai import (
     ChatCompletionRedactedThinkingBlock,
     ChatCompletionThinkingBlock,
     ChatCompletionToolCallChunk,
-    ChatCompletionToolCallFunctionChunk,
-    ChatCompletionUsageBlock,
 )
 from litellm.types.utils import (
-    ChatCompletionMessageToolCall,
-    Choices,
     Delta,
     ModelResponse,
     ModelResponseStream,
@@ -228,7 +225,10 @@ async def make_call(
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
     stream_chunk_size: int | None = None,
     timeout: float | httpx.Timeout | None = None,
-) -> "tuple[MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
+    optional_params: Mapping[str, object] | None = None,
+) -> tuple[
+    base_model_iterator.MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers
+]:
     try:
         if client is None:
             client = get_async_httpx_client(
@@ -264,15 +264,16 @@ async def make_call(
                 model_response=litellm.ModelResponse(),
                 stream=True,
                 logging_obj=logging_obj,
-                optional_params={},
+                optional_params=dict(optional_params or {}),
                 api_key="",
                 data=data,
                 messages=messages,
                 encoding=litellm.encoding,
+                json_mode=json_mode,
             )
-            completion_stream: MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict] = (
-                MockResponseIterator(model_response=model_response, json_mode=json_mode)
-            )
+            completion_stream: (
+                base_model_iterator.MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict]
+            ) = base_model_iterator.MockResponseIterator(model_response=model_response)
         else:
             decoder: Final = _stream_decoder(
                 bedrock_invoke_provider,
@@ -324,7 +325,7 @@ def make_sync_call(
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
     stream_chunk_size: int | None = None,
     timeout: float | httpx.Timeout | None = None,
-) -> "tuple[MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
+) -> tuple[base_model_iterator.MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]:
     try:
         if client is None:
             client = get_httpx_client(
@@ -364,10 +365,11 @@ def make_sync_call(
                 data=data,
                 messages=messages,
                 encoding=litellm.encoding,
+                json_mode=json_mode,
             )
-            completion_stream: MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict] = (
-                MockResponseIterator(model_response=model_response, json_mode=json_mode)
-            )
+            completion_stream: (
+                base_model_iterator.MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict]
+            ) = base_model_iterator.MockResponseIterator(model_response=model_response)
         else:
             decoder: Final = _stream_decoder(
                 bedrock_invoke_provider,
@@ -991,98 +993,3 @@ class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
 
     def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
         return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
-
-
-class MockResponseIterator:  # for returning ai21 streaming responses
-    def __init__(self, model_response, json_mode: bool | None = False):
-        self.model_response = model_response
-        self.json_mode = json_mode
-        self.is_done = False
-
-    # Sync iterator
-    def __iter__(self):
-        return self
-
-    def _handle_json_mode_chunk(
-        self, text: str, tool_calls: list[ChatCompletionToolCallChunk] | None
-    ) -> tuple[str, ChatCompletionToolCallChunk | None]:
-        """
-        If JSON mode is enabled, convert the tool call to a message.
-
-        Bedrock returns the JSON schema as part of the tool call
-        OpenAI returns the JSON schema as part of the content, this handles placing it in the content
-
-        Args:
-            text: str
-            tool_use: Optional[ChatCompletionToolCallChunk]
-        Returns:
-            Tuple[str, Optional[ChatCompletionToolCallChunk]]
-
-            text: The text to use in the content
-            tool_use: The ChatCompletionToolCallChunk to use in the chunk response
-        """
-        tool_use: ChatCompletionToolCallChunk | None = None
-        if self.json_mode is True and tool_calls is not None:
-            message: Final = litellm.AnthropicConfig().convert_tool_response_to_message(tool_calls=tool_calls)
-            if message is not None:
-                text = message.content or ""
-                tool_use = None
-        elif tool_calls is not None and len(tool_calls) > 0:
-            tool_use = tool_calls[0]
-        return text, tool_use
-
-    def _chunk_parser(self, chunk_data: ModelResponse) -> GChunk:
-        try:
-            chunk_usage: Final[Usage] = getattr(chunk_data, "usage")
-            text = chunk_data.choices[0].message.content or ""
-            tool_use = None
-            _model_response_tool_call: Final = cast(
-                list[ChatCompletionMessageToolCall] | None,
-                cast(Choices, chunk_data.choices[0]).message.tool_calls,
-            )
-            if self.json_mode is True:
-                text, tool_use = self._handle_json_mode_chunk(
-                    text=text,
-                    tool_calls=chunk_data.choices[0].message.tool_calls,
-                )
-            elif _model_response_tool_call is not None:
-                tool_use = ChatCompletionToolCallChunk(
-                    id=_model_response_tool_call[0].id,
-                    type="function",
-                    function=ChatCompletionToolCallFunctionChunk(
-                        name=_model_response_tool_call[0].function.name,
-                        arguments=_model_response_tool_call[0].function.arguments,
-                    ),
-                    index=0,
-                )
-            processed_chunk: Final = GChunk(
-                text=text,
-                tool_use=tool_use,
-                is_finished=True,
-                finish_reason=map_finish_reason(finish_reason=chunk_data.choices[0].finish_reason or ""),
-                usage=ChatCompletionUsageBlock(
-                    prompt_tokens=chunk_usage.prompt_tokens,
-                    completion_tokens=chunk_usage.completion_tokens,
-                    total_tokens=chunk_usage.total_tokens,
-                ),
-                index=0,
-            )
-            return processed_chunk
-        except Exception as e:
-            raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}")
-
-    def __next__(self):
-        if self.is_done:
-            raise StopIteration
-        self.is_done = True
-        return self._chunk_parser(self.model_response)
-
-    # Async iterator
-    def __aiter__(self):
-        return self
-
-    async def __anext__(self):
-        if self.is_done:
-            raise StopAsyncIteration
-        self.is_done = True
-        return self._chunk_parser(self.model_response)

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -32,6 +32,7 @@ from litellm.llms.bedrock.common_utils import (
     tools_without_eager_input_streaming,
 )
 from litellm.types.llms.anthropic import (
+    ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
     ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
     AnthropicThinkingParam,
@@ -270,7 +271,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
         is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking)
 
-        user_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
+        client_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
+        user_beta_set: Final = (
+            client_beta_set
+            if "safeguards" in optional_params
+            else client_beta_set - {ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value}
+        )
         beta_set: Final = set(user_beta_set)
         auto_betas: Final = self.get_anthropic_beta_list(
             model=model,

litellm/llms/bedrock/count_tokens/bedrock_token_counter.py

@@ -16,7 +16,13 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
 from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
 from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-from litellm.types.utils import LiteLLMPydanticObjectBase, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import (
+    CountContextManagement,
+    LiteLLMPydanticObjectBase,
+    LlmProviders,
+    TokenCountMessagesFormat,
+    TokenCountResponse,
+)
 
 RUNTIME_TOKENIZER_TYPE: Final = "bedrock_api"
 MANTLE_TOKENIZER_TYPE: Final = "bedrock_mantle_api"
@@ -109,6 +115,7 @@ class BedrockTokenCounter(BaseTokenCounter):
         system: object | None = None,
         messages_format: TokenCountMessagesFormat = "anthropic",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         if not messages:
             return None

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -90,6 +90,9 @@ class AmazonAnthropicClaudeMessagesConfig(
     def beta_headers_provider(self) -> str:
         return self.custom_llm_provider or "bedrock"
 
+    def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
+        return True
+
     BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
 
     def get_error_class(
@@ -552,8 +555,17 @@ class AmazonAnthropicClaudeMessagesConfig(
         if anthropic_model_info.is_eager_input_streaming_used(tools):
             beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
 
+        dangerous_tool_use_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
         if anthropic_messages_optional_request_params.get("safeguards") is not None:
-            beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
+            beta_set.add(dangerous_tool_use_beta)
+        elif dangerous_tool_use_beta in beta_set and self.drops_dangerous_tool_use_beta_without_safeguards():
+            beta_set.discard(dangerous_tool_use_beta)
+            verbose_logger.debug(
+                "Bedrock Invoke: dropping %s from client headers because the request has no safeguards. Bedrock "
+                "answers it with 'invalid beta flag' next to structured outputs before Opus 4.7, and on Sonnet 4.5, "
+                "Opus 4.5, and Opus 4.6 even alone.",
+                dangerous_tool_use_beta,
+            )
 
         self._filter_context_management_for_bedrock_invoke(
             anthropic_messages_request=anthropic_messages_request,

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -67,6 +67,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
     def should_filter_anthropic_beta_headers(self) -> bool:
         return False
 
+    def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
+        return False
+
     def get_complete_url(
         self,
         api_base: str | None,

litellm/llms/bedrock/realtime/handler.py

@@ -8,7 +8,7 @@ import asyncio
 import contextlib
 import importlib.metadata
 import json
-from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping
+from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final, NoReturn, Protocol, runtime_checkable
@@ -21,7 +21,7 @@ from litellm._uuid import uuid
 from litellm.constants import (
     BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
     BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
-    BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY,
+    BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
     BEDROCK_REALTIME_SDK_DISTRIBUTION,
     BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
     BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
@@ -37,9 +37,8 @@ from litellm.types.realtime import RealtimeResponseTransformInput
 
 from ..base_aws_llm import BaseAWSLLM, run_aws_signing
 from ..common_utils import BedrockError
-from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection
+from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
 
-_CLIENT_MODALITIES_ADAPTER: Final[TypeAdapter["list[str] | None"]] = TypeAdapter(list[str] | None)
 _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
 _EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
@@ -177,25 +176,35 @@ async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message:
             return
 
 
-def _pending_session_update(scope: Mapping[str, object]) -> str | None:
-    """A fallback attempt on the same websocket replays the session.update the failed attempt never acked."""
-    if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
-        committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
-        raise BedrockError(
-            status_code=400,
-            message=(
-                "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
-                + (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
-            ),
-        )
-    pending: Final = scope.get(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY)
+def _is_session_committed(scope: Mapping[str, object]) -> bool:
+    return scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True
+
+
+def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
+    """A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
+    if scope is None or not _is_session_committed(scope):
+        return
+    committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
+    raise BedrockError(
+        status_code=400,
+        message=(
+            "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+            + (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
+        ),
+    )
+
+
+def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
+    """A fallback attempt on the same websocket replays the opening message Bedrock never accepted a stream for."""
+    raise_if_session_committed(scope)
+    pending: Final = scope.get(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY)
     return pending if isinstance(pending, str) else None
 
 
 def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
     error: Final = _as_bedrock_error(failure)
     verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
-    if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
+    if _is_session_committed(scope):
         scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
     raise error from failure
 
@@ -207,26 +216,36 @@ def _parse_client_message(message: str) -> Mapping[str, JsonValue]:
         return _EMPTY_JSON_OBJECT
 
 
-async def _ack_session_update(
+async def _send_opening_message(
     client_ws: RealtimeClientWebSocket,
     bedrock_stream: BedrockBidirectionalStream,
+    message: str,
+    bedrock_messages: Sequence[str],
+    send_to_bedrock: Callable[[str], Awaitable[None]],
+) -> None:
+    """Keep the opening message for a fallback to replay until Bedrock accepts the stream it opened."""
+    client_ws.scope[BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY] = message  # rebind-ok: scope outlives attempts
+    for bedrock_message in bedrock_messages:
+        await send_to_bedrock(bedrock_message)
+    await bedrock_stream.await_output()
+    client_ws.scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
+    client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True  # rebind-ok: scope outlives the attempt
+
+
+async def _ack_session_update(
+    client_ws: RealtimeClientWebSocket,
     transformation_config: BedrockRealtimeConfig,
     model: str,
     logging_obj: LiteLLMLogging | None,
     parsed_client_message: Mapping[str, JsonValue],
 ) -> bool:
-    """Ack the client's session.update once Bedrock accepted the stream; False means the client is gone."""
-    await bedrock_stream.await_output()
-    client_ws.scope.pop(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY, None)
-    client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True  # rebind-ok: scope outlives the attempt
+    """Ack the client's session.update; False means the client is gone."""
     if logging_obj is None:
         return True
-    requested_modalities: Final = _CLIENT_MODALITIES_ADAPTER.validate_python(
-        _json_dict(parsed_client_message.get("session")).get("modalities")
-    )
+    modalities: Final = requested_modalities(_json_dict(parsed_client_message.get("session")))
     try:
         await client_ws.send_text(
-            json.dumps(transformation_config.session_updated_event(model, logging_obj, requested_modalities))
+            json.dumps(transformation_config.session_updated_event(model, logging_obj, modalities))
         )
     except Exception as e:  # noqa: BLE001  # any send failure means the client is gone
         verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
@@ -314,7 +333,7 @@ class BedrockRealtime(BaseAWSLLM):
         except ImportError as e:
             raise _sdk_import_error(self._sdk_version_lookup(), e) from e
 
-        pending_session_update: Final = _pending_session_update(websocket.scope)
+        pending_opening_message: Final = _pending_opening_message(websocket.scope)
 
         # Get AWS region
         if aws_region_name is None:
@@ -376,7 +395,7 @@ class BedrockRealtime(BaseAWSLLM):
 
         try:
             await self._run_session(
-                websocket, open_bidirectional_stream, model, logging_obj, pending_session_update, drop_params
+                websocket, open_bidirectional_stream, model, logging_obj, pending_opening_message, drop_params
             )
         finally:
             await _close_bedrock_client(bedrock_client)
@@ -387,7 +406,7 @@ class BedrockRealtime(BaseAWSLLM):
         open_bidirectional_stream: Callable[[], Awaitable[BedrockBidirectionalStream]],
         model: str,
         logging_obj: LiteLLMLogging,
-        pending_session_update: str | None,
+        pending_opening_message: str | None,
         drop_params: bool = False,
     ) -> None:
         transformation_config: Final = BedrockRealtimeConfig(drop_params=drop_params)
@@ -396,7 +415,7 @@ class BedrockRealtime(BaseAWSLLM):
 
         verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
 
-        if pending_session_update is None:
+        if pending_opening_message is None:
             await websocket.send_text(json.dumps(transformation_config.session_created_event(model, logging_obj)))
             verbose_proxy_logger.debug("Bedrock Realtime: sent session.created to client on connect")
 
@@ -418,7 +437,7 @@ class BedrockRealtime(BaseAWSLLM):
             model,
             session_state,
             logging_obj,
-            initial_message=pending_session_update,
+            initial_message=pending_opening_message,
         )
 
         logged_events: Final = (
@@ -531,18 +550,22 @@ class BedrockRealtime(BaseAWSLLM):
                         if not await _reject_session_update(client_ws, parsed_client_message, rejection):
                             break
                         continue
-                    client_ws.scope[BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY] = message
 
                 transformed_messages = transformation_config.transform_realtime_request(
                     message=message,
                     model=model,
                     session_configuration_request=session_state.get("session_configuration_request"),
                 )
-                for bedrock_message in transformed_messages:
-                    await send_to_bedrock(bedrock_message)
+                if transformed_messages and not _is_session_committed(client_ws.scope):
+                    await _send_opening_message(
+                        client_ws, bedrock_stream, message, transformed_messages, send_to_bedrock
+                    )
+                else:
+                    for bedrock_message in transformed_messages:
+                        await send_to_bedrock(bedrock_message)
 
                 if is_session_update and not await _ack_session_update(
-                    client_ws, bedrock_stream, transformation_config, model, logging_obj, parsed_client_message
+                    client_ws, transformation_config, model, logging_obj, parsed_client_message
                 ):
                     break
         finally:

litellm/llms/bedrock/realtime/transformation.py

@@ -123,26 +123,81 @@ def _integer_setting(value: JsonValue) -> int | None:
     return None if number is None else int(number)
 
 
-def _malformed_number_settings(session_config: Mapping[str, JsonValue]) -> tuple[str, ...]:
+def requested_modalities(session_config: Mapping[str, JsonValue]) -> tuple[str, ...] | None:
+    modalities: Final = session_config.get("modalities")
+    if not isinstance(modalities, list):
+        return None
+    names: Final = tuple(modality for modality in modalities if isinstance(modality, str))
+    return names if len(names) == len(modalities) else None
+
+
+_OPENAI_VOICES: Final = frozenset(
+    ("alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse", "marin", "cedar")
+)
+
+
+def _unplayable_voice(session_config: Mapping[str, JsonValue]) -> str | None:
+    voice: Final = session_config.get("voice")
+    if not isinstance(voice, str):
+        return None
+    return voice if not voice.strip() or voice.lower() in _OPENAI_VOICES else None
+
+
+def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[str, str], ...]:
+    instructions: Final = session_config.get("instructions")
+    voice: Final = session_config.get("voice")
+    input_audio_format: Final = session_config.get("input_audio_format")
+    output_audio_format: Final = session_config.get("output_audio_format")
+    modalities: Final = session_config.get("modalities")
     temperature: Final = session_config.get("temperature")
     max_output_tokens: Final = session_config.get("max_response_output_tokens")
+    input_rate_hertz: Final = session_config.get("input_sample_rate_hertz")
     output_rate_hertz: Final = session_config.get("output_sample_rate_hertz")
+    tools: Final = session_config.get("tools")
     malformed: Final = (
-        ("temperature", temperature is not None and _number_setting(temperature) is None),
+        ("instructions", "a string", instructions is not None and not isinstance(instructions, str)),
+        ("voice", "a string", voice is not None and not isinstance(voice, str)),
+        ("tools", "a list", tools is not None and not isinstance(tools, list)),
+        (
+            "input_audio_format",
+            "a string",
+            input_audio_format is not None and not isinstance(input_audio_format, str),
+        ),
+        (
+            "output_audio_format",
+            "a string",
+            output_audio_format is not None and not isinstance(output_audio_format, str),
+        ),
+        (
+            "modalities",
+            "a list of strings",
+            modalities is not None and requested_modalities(session_config) is None,
+        ),
+        ("temperature", "a number", temperature is not None and _number_setting(temperature) is None),
         (
             "max_response_output_tokens",
+            "a number or 'inf'",
             max_output_tokens is not None
             and max_output_tokens != "inf"
             and _integer_setting(max_output_tokens) is None,
         ),
-        ("output_sample_rate_hertz", output_rate_hertz is not None and _integer_setting(output_rate_hertz) is None),
+        (
+            "input_sample_rate_hertz",
+            "a number",
+            input_rate_hertz is not None and _integer_setting(input_rate_hertz) is None,
+        ),
+        (
+            "output_sample_rate_hertz",
+            "a number",
+            output_rate_hertz is not None and _integer_setting(output_rate_hertz) is None,
+        ),
     )
-    return tuple(name for name, is_malformed in malformed if is_malformed)
+    return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
 
 
 @dataclasses.dataclass(frozen=True, slots=True)
 class BedrockSessionUpdateRejection:
-    code: Literal["invalid_type", "session_update_not_applied"]
+    code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
     message: str
     param: str
 
@@ -312,7 +367,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             return 8000  # G.711 typically uses 8kHz
         return 24000 if is_output else 16000
 
-    def transform_session_update_event(self, json_message: dict) -> list[str]:
+    def transform_session_update_event(self, json_message: Mapping[str, JsonValue]) -> list[str]:
         """
         Transform session.update event to Bedrock session configuration.
 
@@ -325,7 +380,10 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         verbose_logger.debug("Handling session.update")
         messages: Final[list[str]] = []
 
-        session_config: Final = json_message.get("session", {})
+        requested_session: Final = json_message.get("session")
+        session_config: Final[Mapping[str, JsonValue]] = (
+            requested_session if isinstance(requested_session, dict) else {}
+        )
         locked_settings: Final = self._apply_locked_settings(session_config, self._started_settings)
 
         # Settings Bedrock locks at prompt start may only change before it
@@ -336,13 +394,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             self.output_sample_rate_hertz = locked_settings.output_sample_rate_hertz
 
         # Update audio input configuration from session if provided
-        if "input_audio_format" in session_config:
-            input_format: Final = session_config["input_audio_format"]
+        input_format: Final = session_config.get("input_audio_format")
+        if isinstance(input_format, str):
             self.input_sample_rate_hertz = self._map_audio_format_to_sample_rate(input_format, is_output=False)
 
         # Allow direct override of the input sample rate if provided (custom extension)
-        if "input_sample_rate_hertz" in session_config:
-            self.input_sample_rate_hertz = session_config["input_sample_rate_hertz"]
+        requested_input_rate: Final = _integer_setting(session_config.get("input_sample_rate_hertz"))
+        if requested_input_rate is not None:
+            self.input_sample_rate_hertz = requested_input_rate
 
         if self.prompt_started:
             return messages
@@ -352,7 +411,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         messages.append(json.dumps(session_start))
 
         # Send prompt start
-        tools: Final[Sequence[JsonValue] | None] = session_config.get("tools")
+        tools: Final = session_config.get("tools")
         prompt_start_config: Final[Mapping[str, object]] = {
             "promptName": self.prompt_name,
             "textOutputConfiguration": {"mediaType": self.text_media_type},
@@ -370,7 +429,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                     "toolUseOutputConfiguration": {"mediaType": "application/json"},
                     "toolConfiguration": {"tools": self._transform_tools_to_bedrock_format(tools)},
                 }
-                if tools
+                if isinstance(tools, list) and tools
                 else {}
             ),
         }
@@ -445,12 +504,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             if isinstance(requested_tools, list)
             else base.tools_json
         )
-        system_prompt: Final = base.system_prompt if instructions is None else str(instructions)
-        voice_id: Final = base.voice_id if voice is None else str(voice)
+        system_prompt: Final = instructions if isinstance(instructions, str) else base.system_prompt
+        voice_id: Final = (
+            voice if isinstance(voice, str) and _unplayable_voice(session_config) is None else base.voice_id
+        )
         format_sample_rate: Final = (
-            base.output_sample_rate_hertz
-            if output_audio_format is None
-            else self._map_audio_format_to_sample_rate(str(output_audio_format), is_output=True)
+            self._map_audio_format_to_sample_rate(output_audio_format, is_output=True)
+            if isinstance(output_audio_format, str)
+            else base.output_sample_rate_hertz
         )
         requested_output_rate: Final = _integer_setting(session_config.get("output_sample_rate_hertz"))
         output_sample_rate_hertz: Final = (
@@ -476,16 +537,26 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         session_config: Final = json_message.get("session")
         if not isinstance(session_config, Mapping):
             return None
-        malformed: Final = () if self.drop_params else _malformed_number_settings(session_config)
+        malformed: Final = () if self.drop_params else _malformed_settings(session_config)
         if malformed:
+            name, expected = malformed[0]
             return BedrockSessionUpdateRejection(
                 code="invalid_type",
                 message=(
-                    f"Invalid type for 'session.{malformed[0]}': Bedrock Nova Sonic needs a number for temperature "
-                    "and output_sample_rate_hertz, and a number or 'inf' for max_response_output_tokens. "
+                    f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
                     "Set drop_params to drop it instead"
                 ),
-                param=f"session.{malformed[0]}",
+                param=f"session.{name}",
+            )
+        unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
+        if unplayable_voice is not None:
+            return BedrockSessionUpdateRejection(
+                code="invalid_value",
+                message=(
+                    f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
+                    "Set drop_params to drop it instead"
+                ),
+                param="session.voice",
             )
         if not self.prompt_started:
             return None
@@ -785,7 +856,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             List of Bedrock format messages (JSON strings)
         """
         try:
-            json_message: Final[dict[str, object]] = json.loads(message)
+            json_message: Final[dict[str, JsonValue]] = json.loads(message)
         except json.JSONDecodeError:
             verbose_logger.warning("Invalid JSON message: %s", message[:200])
             return []
@@ -816,11 +887,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         self,
         model: str,
         logging_obj: LiteLLMLoggingObj,
-        modalities: list[str] | None = None,
+        modalities: Sequence[str] | None = None,
     ) -> OpenAIRealtimeStreamSession:
         session: Final = OpenAIRealtimeStreamSession(
             id=logging_obj.litellm_trace_id,
-            modalities=modalities if modalities is not None else ["text", "audio"],
+            modalities=list(modalities) if modalities is not None else ["text", "audio"],
         )
         if model is not None and isinstance(model, str):
             session["model"] = model
@@ -842,7 +913,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         self,
         model: str,
         logging_obj: LiteLLMLoggingObj,
-        modalities: list[str] | None = None,
+        modalities: Sequence[str] | None = None,
     ) -> OpenAIRealtimeStreamSessionEvents:
         """Build the OpenAI session.updated ack reflecting the client's requested modalities."""
         return OpenAIRealtimeStreamSessionEvents(

litellm/llms/custom_httpx/container_handler.py

@@ -5,6 +5,7 @@ This module provides a single generic handler that can process any container fil
 endpoint defined in endpoints.json, eliminating the need for individual handler methods.
 """
 
+import asyncio
 import json
 from collections.abc import Coroutine, Mapping, Sequence
 from pathlib import Path
@@ -236,6 +237,7 @@ def _request_headers(
     return container_provider_config.validate_environment(
         headers=extra_headers or {},
         api_key=litellm_params.get("api_key", None),
+        litellm_params=litellm_params,
     )
 
 
@@ -435,7 +437,7 @@ class GenericContainerHandler:
         http_client: Final = _async_http_client(client, litellm_params)
 
         # Build request
-        headers = _request_headers(container_provider_config, extra_headers, litellm_params)
+        headers = await asyncio.to_thread(_request_headers, container_provider_config, extra_headers, litellm_params)
         if extra_headers:
             headers.update(extra_headers)
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -7127,7 +7127,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = image_edit_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            image_edit_provider_config.validate_environment,
             api_key=litellm_params.api_key,
             headers=image_edit_optional_request_params.get("extra_headers", {}) or {},
             model=model,
@@ -7586,7 +7587,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_generation_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_generation_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=video_generation_optional_request_params.get("extra_headers", {}) or {},
             model=model,
@@ -7781,7 +7783,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_content_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_content_provider_config.validate_environment,
             headers=extra_headers or {},
             model="",
             api_key=api_key,
@@ -7961,7 +7964,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_remix_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_remix_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8124,7 +8128,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8264,7 +8269,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8436,7 +8442,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8622,7 +8629,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_provider_config.validate_environment,
             api_key=api_key or litellm_params.get("api_key", None),
             headers=extra_headers or {},
             model="",
@@ -8754,7 +8762,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_list_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_list_provider_config.validate_environment,
             api_key=api_key,
             headers=extra_headers or {},
             model="",
@@ -8835,7 +8844,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_delete_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_delete_provider_config.validate_environment,
             api_key=api_key,
             headers=extra_headers or {},
             model="",
@@ -9019,7 +9029,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = video_status_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            video_status_provider_config.validate_environment,
             api_key=api_key,
             headers=extra_headers or {},
             model="",
@@ -9122,6 +9133,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         # Add Content-Type header for JSON requests
@@ -9197,9 +9209,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         # Add Content-Type header for JSON requests
@@ -9293,6 +9307,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9369,9 +9384,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9460,6 +9477,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9536,9 +9554,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9629,6 +9649,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9705,9 +9726,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9803,6 +9826,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9881,9 +9905,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -9972,6 +9998,7 @@ class BaseLLMHTTPHandler:
         headers: Final = container_provider_config.validate_environment(
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10047,9 +10074,11 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Validate environment and get headers
-        headers: Final = container_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            container_provider_config.validate_environment,
             headers=extra_headers or {},
             api_key=litellm_params.get("api_key", None),
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10167,8 +10196,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10410,8 +10441,10 @@ class BaseLLMHTTPHandler:
 
         vector_store_provider_config.validate_create_vector_store()
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10541,8 +10574,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10663,8 +10698,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10805,8 +10842,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -10950,8 +10989,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -11068,8 +11109,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -11210,8 +11253,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11351,8 +11396,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11482,8 +11529,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11619,8 +11668,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)
@@ -11762,8 +11813,10 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = vector_store_files_provider_config.validate_environment(
-            headers=extra_headers or {}, litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            vector_store_files_provider_config.validate_environment,
+            headers=extra_headers or {},
+            litellm_params=litellm_params,
         )
         if extra_headers:
             headers.update(extra_headers)

litellm/llms/gemini/common_utils.py

@@ -14,7 +14,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
 
 GEMINI_IMAGE_ASPECT_RATIOS: Final[dict[str, float]] = {
     "1:1": 1 / 1,
@@ -497,12 +497,14 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
         system: object | None = None,
         messages_format: TokenCountMessagesFormat = "openai",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         import copy
 
         from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
         from litellm.llms.vertex_ai.count_tokens.context import (
             VertexGeminiCountContextError,
+            count_context_edits,
             count_context_error_response,
             gemini_count_contents,
             inference_request_view,
@@ -511,7 +513,14 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
             vertex_gemini_count_context,
         )
 
-        request_view: Final = inference_request_view(messages or [], system, messages_format)
+        request_view: Final = await inference_request_view(
+            messages or [],
+            system,
+            messages_format,
+            count_context_edits(context_management, tools, (deployment or {}).get("litellm_params", {}), model_to_use),
+        )
+        if isinstance(request_view, VertexGeminiCountContextError):
+            return count_context_error_response(request_view, request_model, model_to_use)
         count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
         if isinstance(count_context, VertexGeminiCountContextError):
             return count_context_error_response(count_context, request_model, model_to_use)

litellm/llms/gemini/realtime/transformation.py

@@ -161,6 +161,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
         return True
 
+    def _rejects_vad_tuning_with_detection_off(self) -> bool:
+        """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
+        return True
+
     @staticmethod
     def _usage_detail_alias(details: Mapping[str, int | None] | None, defaults: dict[str, int]) -> dict[str, int]:
         if not isinstance(details, dict):
@@ -279,10 +283,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
 
         OpenAI ``semantic_vad`` has no Gemini Live equivalent — return an empty
         dict so callers omit ``realtimeInputConfig`` (mapping it with
-        ``disabled: true`` breaks native-audio sessions).
+        ``disabled: true`` breaks native-audio sessions). An explicit
+        ``create_response: false`` maps to detection off alone where the backend
+        rejects tuning next to ``disabled: true``.
         """
         if isinstance(value, dict) and value.get("type") == "semantic_vad" and "create_response" not in value:
             return AutomaticActivityDetection()
+        auto_response_off: Final = "create_response" in value and value["create_response"] is False
+        if auto_response_off and self._rejects_vad_tuning_with_detection_off():
+            return AutomaticActivityDetection(disabled=True)
 
         automatic_activity_dection: Final = AutomaticActivityDetection()
         if "create_response" in value and isinstance(value["create_response"], bool):

litellm/llms/meta/realtime/transformation.py

@@ -26,6 +26,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import (
     speech_event,
     transcription_session,
     transcription_session_created_event,
+    uses_server_vad,
 )
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
 from litellm.secret_managers.main import get_secret_str
@@ -230,15 +231,7 @@ def _parse_sample_rate(update: TranscriptionSessionUpdate) -> MuseSampleRate:
     return 16_000 if rate == 16_000 else 24_000
 
 
-def _parse_mode(update: TranscriptionSessionUpdate) -> MuseMode:
-    if update.turn_detection_disabled:
-        return "PUSH_TO_TALK"
-    if update.turn_detection_type not in (None, "server_vad"):
-        raise MuseProtocolError("Muse Voice supports server_vad turn detection or null")
-    return "ENDPOINTING"
-
-
-def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig:
+def parse_session_update(payload: str, expected_model: str, *, drop_params: bool = False) -> MuseSessionConfig:
     update: Final = parse_transcription_session_update(payload, MuseProtocolError)
     if update.session_type not in (None, "transcription", "realtime"):
         raise MuseProtocolError("Muse Voice supports transcription sessions only")
@@ -253,7 +246,7 @@ def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig
         raise MuseProtocolError("realtime session model cannot be changed")
     return MuseSessionConfig(
         model=normalized_model,
-        mode=_parse_mode(update),
+        mode="ENDPOINTING" if uses_server_vad(update, "Muse Voice", drop_params, MuseProtocolError) else "PUSH_TO_TALK",
         sample_rate=_parse_sample_rate(update),
         language_bias=() if update.language is None else (normalize_language(update.language),),
     )
@@ -441,9 +434,11 @@ class MetaRealtimeConfig(BaseRealtimeConfig):
         *,
         monotonic: Callable[[], float] = time.monotonic,
         sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        drop_params: bool = False,
     ) -> None:
         self._monotonic: Final = monotonic
         self._sleep: Final = sleep
+        self._drop_params: Final = drop_params
         self._transformer: Final = MuseEventTransformer()
         self._access_token: str | None = None
         self._config: MuseSessionConfig | None = None
@@ -553,7 +548,7 @@ class MetaRealtimeConfig(BaseRealtimeConfig):
         access_token: Final = self._access_token
         if access_token is None:
             raise MuseProtocolError("Meta API key was not validated before the session was configured")
-        config: Final = parse_session_update(message, model)
+        config: Final = parse_session_update(message, model, drop_params=self._drop_params)
         self._config = config
         self._transformer.configure(config)
         return (json.dumps(config.handshake(access_token), separators=(",", ":")),)

litellm/llms/nace/decisions/transformation.py

@@ -0,0 +1,9 @@
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class NaceDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("NACE_API_KEY",)
+    api_base_env = ("NACE_API_BASE", "NACE_BASE_URL")
+
+    def get_default_api_base(self) -> str | None:
+        return "https://console.nace.ai"

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -39,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
     effective_skip_tool_message_for_guardrail,
+    list_field,
     merge_guardrailed_scoped_messages,
     merge_returned_tools_into_request_tools,
     openai_tool_name,
@@ -156,7 +157,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices]
             # Pass tools (function definitions) to the guardrail
             tools: Final = data.get("tools")
-            if tools and not scan_only_tool_results:
+            if isinstance(tools, list) and tools and not scan_only_tool_results:
                 inputs["tools"] = tools
             # Include model information if available
             model: Final = data.get("model")
@@ -258,19 +259,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             return "no scannable content after message scoping"
         return None if images else "no scannable content"
 
-    def extract_request_tool_names(self, data: dict) -> list[str]:
+    def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
         """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name,
         and the tools a replayed tool_search_output loads)."""
         names: Final[list[str]] = []
-        for tool in data.get("tools") or []:
+        for tool in list_field(data, "tools"):
             if isinstance(tool, dict) and tool.get("type") == "function":
                 fn = tool.get("function")
                 if isinstance(fn, dict) and fn.get("name"):
                     names.append(str(fn["name"]))
-        for fn in data.get("functions") or []:
+        for fn in list_field(data, "functions"):
             if isinstance(fn, dict) and fn.get("name"):
                 names.append(str(fn["name"]))
-        return [*names, *replayed_tool_search_tool_names(cast(object, data))]  # cast-ok: upcast, read field by field
+        return [*names, *replayed_tool_search_tool_names(data)]
 
     def _extract_inputs(
         self,

litellm/llms/openai/common_utils.py

@@ -407,6 +407,10 @@ class BaseOpenAILLM:
             http2=http2_enabled(),
         )
 
+    @classmethod
+    def get_sync_http_client(cls) -> httpx.Client | None:
+        return cls._get_sync_http_client()
+
 
 class OpenAICredentials(NamedTuple):
     api_base: str

litellm/llms/openai/completion/handler.py

@@ -1,9 +1,7 @@
 import json
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
 
-from openai import AsyncOpenAI, OpenAI
-
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@@ -12,7 +10,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
 from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
 from litellm.utils import ProviderConfigManager
 
-from ..common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
+from ..common_utils import BaseOpenAILLM, OpenAIError
+from ..workload_identity import build_async_openai_client, build_openai_client
 from .transformation import OpenAITextCompletionConfig
 
 
@@ -43,7 +42,7 @@ class OpenAITextCompletion(BaseLLM):
         print_verbose: Callable | None = None,
         api_base: str | None = None,
         acompletion: bool = False,
-        litellm_params=None,
+        litellm_params: Mapping[str, object] | None = None,
         logger_fn=None,
         client=None,
         organization: str | None = None,
@@ -95,6 +94,7 @@ class OpenAITextCompletion(BaseLLM):
                         max_retries=max_retries,
                         client=client,
                         organization=organization,
+                        litellm_params=litellm_params,
                     )
                 else:
                     return self.acompletion(
@@ -109,6 +109,7 @@ class OpenAITextCompletion(BaseLLM):
                         max_retries=max_retries,
                         organization=organization,
                         client=client,
+                        litellm_params=litellm_params,
                     )
             elif optional_params.get("stream", False):
                 return self.streaming(
@@ -123,19 +124,22 @@ class OpenAITextCompletion(BaseLLM):
                     max_retries=max_retries,
                     client=client,
                     organization=organization,
+                    litellm_params=litellm_params,
                 )
             else:
-                if client is None:
-                    openai_client = OpenAI(
+                openai_client: Final = (
+                    client
+                    if client is not None
+                    else build_openai_client(
                         api_key=api_key,
-                        base_url=api_base,
-                        http_client=litellm.client_session or OpenAIHTTPClient(),
+                        api_base=api_base,
                         timeout=timeout,
                         max_retries=max_retries,
                         organization=organization,
+                        litellm_params=litellm_params,
+                        static_key_http_client_factory=lambda: litellm.client_session,
                     )
-                else:
-                    openai_client = client
+                )
 
                 raw_response: Final = openai_client.completions.with_raw_response.create(**data)
                 response: Final = raw_response.parse()
@@ -175,19 +179,22 @@ class OpenAITextCompletion(BaseLLM):
         max_retries: int,
         organization: str | None = None,
         client=None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         try:
-            if client is None:
-                openai_aclient = AsyncOpenAI(
+            openai_aclient: Final = (
+                client
+                if client is not None
+                else build_async_openai_client(
                     api_key=api_key,
-                    base_url=api_base,
-                    http_client=BaseOpenAILLM.get_async_http_client(),
+                    api_base=api_base,
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
+                    litellm_params=litellm_params,
+                    static_key_http_client_factory=BaseOpenAILLM.get_async_http_client,
                 )
-            else:
-                openai_aclient = client
+            )
 
             raw_response: Final = await openai_aclient.completions.with_raw_response.create(**data)
             response: Final = raw_response.parse()
@@ -228,18 +235,21 @@ class OpenAITextCompletion(BaseLLM):
         max_retries=None,
         client=None,
         organization=None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
-        if client is None:
-            openai_client = OpenAI(
+        openai_client: Final = (
+            client
+            if client is not None
+            else build_openai_client(
                 api_key=api_key,
-                base_url=api_base,
-                http_client=litellm.client_session or OpenAIHTTPClient(),
+                api_base=api_base,
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
+                static_key_http_client_factory=lambda: litellm.client_session,
             )
-        else:
-            openai_client = client
+        )
 
         try:
             raw_response: Final = openai_client.completions.with_raw_response.create(**data)
@@ -285,18 +295,21 @@ class OpenAITextCompletion(BaseLLM):
         api_base: str | None = None,
         client=None,
         organization=None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
-        if client is None:
-            openai_client = AsyncOpenAI(
+        openai_client: Final = (
+            client
+            if client is not None
+            else build_async_openai_client(
                 api_key=api_key,
-                base_url=api_base,
-                http_client=litellm.aclient_session or OpenAIAsyncHTTPClient(),
+                api_base=api_base,
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
+                static_key_http_client_factory=lambda: litellm.aclient_session,
             )
-        else:
-            openai_client = client
+        )
 
         raw_response: Final = await openai_client.completions.with_raw_response.create(**data)
         response: Final = raw_response.parse()

litellm/llms/openai/containers/transformation.py

@@ -23,6 +23,7 @@ from litellm.types.containers.main import (
 from litellm.types.router import GenericLiteLLMParams
 
 from ...base_llm.containers.transformation import BaseContainerConfig
+from ..workload_identity import resolve_openai_bearer_token
 from .utils import join_container_api_base_path
 
 if TYPE_CHECKING:
@@ -104,13 +105,15 @@ class OpenAIContainerConfig(BaseContainerConfig):
         self,
         headers: dict,
         api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            },
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=api_key or params.api_key,
+            api_base=params.api_base,
+            litellm_params=params.model_dump(exclude_none=True),
         )
+        headers.update({"Authorization": f"Bearer {bearer_token}"})
         return headers
 
     def get_complete_url(

litellm/llms/openai/decisions/transformation.py

@@ -6,6 +6,7 @@ from typing_extensions import assert_never
 
 import litellm
 from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.decisions import (
     DecisionsImage,
@@ -349,6 +350,11 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
     def resolve_api_key(self, api_key: str | None) -> str | None:
         return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
 
+    def resolve_credential(
+        self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
+    ) -> str | None:
+        return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
+
     def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
         return False
 

litellm/llms/openai/evals/transformation.py

@@ -13,6 +13,7 @@ from litellm.llms.base_llm.evals.transformation import (
     BaseEvalsAPIConfig,
     LiteLLMLoggingObj,
 )
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.types.llms.openai_evals import (
     CancelEvalResponse,
     CancelRunResponse,
@@ -44,24 +45,14 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         return LlmProviders.OPENAI
 
     def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
-        """Add OpenAI-specific headers"""
-        import litellm
-        from litellm.secret_managers.main import get_secret_str
-
-        # Get API key following OpenAI pattern
-        api_key = None
-        if litellm_params:
-            api_key = litellm_params.api_key
-
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-
-        if not api_key:
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
+        )
+        if not bearer_token:
             raise ValueError("OPENAI_API_KEY is required for Evals API")
-
-        # Add required headers
-        headers["Authorization"] = f"Bearer {api_key}"
+        headers["Authorization"] = f"Bearer {bearer_token}"
         headers["Content-Type"] = "application/json"
-
         return headers
 
     def get_complete_url(

litellm/llms/openai/fine_tuning/handler.py

@@ -6,7 +6,7 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 from openai.types.fine_tuning import FineTuningJob
 
 from litellm._logging import verbose_logger
-from litellm.llms.openai.common_utils import OpenAIAsyncHTTPClient, OpenAIHTTPClient
+from litellm.llms.openai.workload_identity import build_async_openai_client, build_openai_client
 from litellm.types.utils import LiteLLMFineTuningJob
 
 _AZURE_STATUS_MAP: Final[Mapping[object, str]] = {
@@ -74,27 +74,27 @@ class OpenAIFineTuningAPI:
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
         _is_async: bool = False,
         api_version: str | None = None,
-        litellm_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
-        received_args: Final = locals()
-        openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client" or k == "_is_async":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            if _is_async is True:
-                openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-            else:
-                openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        if _is_async:
+            return build_async_openai_client(
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     async def acreate_fine_tuning_job(
         self,
@@ -116,6 +116,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -126,6 +127,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -164,6 +166,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -174,6 +177,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -212,6 +216,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
         after: str | None = None,
         limit: int | None = None,
     ):
@@ -224,6 +229,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -263,6 +269,7 @@ class OpenAIFineTuningAPI:
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
         openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -273,6 +280,7 @@ class OpenAIFineTuningAPI:
             client=client,
             _is_async=_is_async,
             api_version=api_version,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(

litellm/llms/openai/image_edit/transformation.py

@@ -17,6 +17,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import ImageResponse
 
 from ..common_utils import OpenAIError
+from ..workload_identity import resolve_openai_bearer_token
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -160,12 +161,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
         litellm_params: dict | None = None,
         api_base: str | None = None,
     ) -> dict:
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            }
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=api_key, api_base=api_base, litellm_params=litellm_params
         )
+        headers.update({"Authorization": f"Bearer {bearer_token}"})
         return headers
 
     def get_complete_url(

litellm/llms/openai/image_variations/handler.py

@@ -2,7 +2,7 @@
 OpenAI Image Variations Handler
 """
 
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
 
 import httpx
@@ -14,27 +14,58 @@ from litellm.utils import ProviderConfigManager
 
 from ...base_llm.image_variations.transformation import BaseImageVariationConfig
 from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
-from ..common_utils import OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
+from ..common_utils import OpenAIError
+from ..workload_identity import build_async_openai_client, build_openai_client
 
 
 class OpenAIImageVariationsHandler:
     def get_sync_client(
         self,
         client: OpenAI | None,
-        init_client_params: dict,
-    ):
+        api_key: str | None,
+        api_base: str,
+        timeout: float | httpx.Timeout | None,
+        max_retries: int,
+        organization: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> OpenAI:
         if client is not None:
             return client
-        return OpenAI(**init_client_params, http_client=litellm.client_session or OpenAIHTTPClient())
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+            static_key_http_client_factory=lambda: litellm.client_session,
+        )
 
-    def get_async_client(self, client: AsyncOpenAI | None, init_client_params: dict) -> AsyncOpenAI:
+    def get_async_client(
+        self,
+        client: AsyncOpenAI | None,
+        api_key: str | None,
+        api_base: str,
+        timeout: float | httpx.Timeout | None,
+        max_retries: int,
+        organization: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> AsyncOpenAI:
         if client is not None:
             return client
-        return AsyncOpenAI(**init_client_params, http_client=litellm.aclient_session or OpenAIAsyncHTTPClient())
+        return build_async_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+            static_key_http_client_factory=lambda: litellm.aclient_session,
+        )
 
     async def async_image_variations(
         self,
-        api_key: str,
+        api_key: str | None,
         api_base: str,
         organization: str | None,
         client: AsyncOpenAI | None,
@@ -51,15 +82,15 @@ class OpenAIImageVariationsHandler:
         provider_config: BaseImageVariationConfig,
     ) -> ImageResponse:
         try:
-            init_client_params: Final = {
-                "api_key": api_key,
-                "base_url": api_base,
-                "timeout": timeout,
-                "max_retries": max_retries,
-                "organization": organization,
-            }
-
-            client = self.get_async_client(client=client, init_client_params=init_client_params)
+            client = self.get_async_client(
+                client=client,
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
 
             raw_response: Final = await client.images.with_raw_response.create_variation(**data)
             response: Final = raw_response.parse()
@@ -103,7 +134,7 @@ class OpenAIImageVariationsHandler:
     def image_variations(
         self,
         model_response: ImageResponse,
-        api_key: str,
+        api_key: str | None,
         api_base: str,
         model: str | None,
         image: FileTypes,
@@ -167,15 +198,15 @@ class OpenAIImageVariationsHandler:
                     litellm_params=litellm_params,
                 )
 
-            init_client_params: Final = {
-                "api_key": api_key,
-                "base_url": api_base,
-                "timeout": timeout,
-                "max_retries": max_retries,
-                "organization": organization,
-            }
-
-            client = self.get_sync_client(client=client, init_client_params=init_client_params)
+            client = self.get_sync_client(
+                client=client,
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
 
             raw_response: Final = client.images.with_raw_response.create_variation(**json_data)
             response: Final = raw_response.parse()

litellm/llms/openai/openai.py

@@ -11,13 +11,14 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
 import openai
-from openai import AsyncOpenAI, OpenAI
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
 from openai._base_client import make_request_options
 from openai._constants import RAW_RESPONSE_HEADER
 from openai._legacy_response import LegacyAPIResponse
 from openai._types import RequestOptions
 from openai.types import CreateEmbeddingResponse
 from openai.types.beta.assistant_deleted import AssistantDeleted
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.file_deleted import FileDeleted
 from pydantic import BaseModel, TypeAdapter
 from typing_extensions import overload
@@ -30,9 +31,8 @@ from litellm.files.types import FileContentStreamingResult
 from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
-from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
-from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
 from litellm.types.utils import (
     EmbeddingResponse,
     ImageResponse,
@@ -53,16 +53,19 @@ from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig
 from .chat.o_series_transformation import OpenAIOSeriesConfig
 from .common_utils import (
     BaseOpenAILLM,
-    OpenAIAsyncHTTPClient,
     OpenAIError,
-    OpenAIHTTPClient,
     build_output_token_limit_response,
     drop_params_from_unprocessable_entity_error,
     is_openai_backed_api_base,
     is_output_token_limit_error,
     model_dump_provider_response,
 )
-from .workload_identity import resolve_openai_workload_identity_config
+from .workload_identity import (
+    build_async_openai_client,
+    build_openai_client,
+    openai_static_api_key,
+    resolve_openai_workload_identity_config,
+)
 
 openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
 openAIGPT5Config: Final = OpenAIGPT5Config()
@@ -352,7 +355,14 @@ _NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
 _SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body"))
 
 
-def _embedding_request_without_sdk_defaults(
+def _raw_chat_response(
+    result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk],
+) -> LegacyAPIResponse[ChatCompletion]:
+    assert isinstance(result, LegacyAPIResponse)
+    return result
+
+
+def _request_without_sdk_transform(
     data: Mapping[str, object], timeout: float | httpx.Timeout
 ) -> tuple[Mapping[str, object], RequestOptions]:
     body: Final = {k: v for k, v in data.items() if k not in _SDK_OPTION_KEYS}
@@ -426,7 +436,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     )
                     if workload_identity_config is not None
                     else AsyncOpenAI(
-                        api_key=api_key,
+                        api_key=openai_static_api_key(api_key),
                         base_url=api_base,
                         http_client=async_http_client,
                         timeout=timeout,
@@ -448,7 +458,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     )
                     if workload_identity_config is not None
                     else OpenAI(
-                        api_key=api_key,
+                        api_key=openai_static_api_key(api_key),
                         base_url=api_base,
                         http_client=sync_http_client,
                         timeout=timeout,
@@ -515,7 +525,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         """
         start_time: Final = time.time()
         try:
-            raw_response = await openai_aclient.chat.completions.with_raw_response.create(**data, timeout=timeout)
+            body, options = _request_without_sdk_transform(data, timeout)
+            raw_response = _raw_chat_response(
+                await openai_aclient.post(
+                    "/chat/completions",
+                    body=body,
+                    cast_to=ChatCompletion,
+                    options=options,
+                    stream=bool(data.get("stream")),
+                    stream_cls=AsyncStream[ChatCompletionChunk],
+                )
+            )
             end_time = time.time()
 
             if hasattr(raw_response, "headers"):
@@ -556,7 +576,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         """
         raw_response = None
         try:
-            raw_response = openai_client.chat.completions.with_raw_response.create(**data, timeout=timeout)
+            body, options = _request_without_sdk_transform(data, timeout)
+            raw_response = _raw_chat_response(
+                openai_client.post(
+                    "/chat/completions",
+                    body=body,
+                    cast_to=ChatCompletion,
+                    options=options,
+                    stream=bool(data.get("stream")),
+                    stream_cls=Stream[ChatCompletionChunk],
+                )
+            )
 
             if hasattr(raw_response, "headers"):
                 headers = dict(raw_response.headers)
@@ -1255,7 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         logging_obj: LiteLLMLoggingObj,
     ) -> LegacyAPIResponse[CreateEmbeddingResponse]:
         if "encoding_format" not in data:
-            body, options = _embedding_request_without_sdk_defaults(data, timeout)
+            body, options = _request_without_sdk_transform(data, timeout)
             bypass_response: Final = await openai_aclient.post(
                 "/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
             )
@@ -1272,7 +1302,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         logging_obj: LiteLLMLoggingObj,
     ) -> LegacyAPIResponse[CreateEmbeddingResponse]:
         if "encoding_format" not in data:
-            body, options = _embedding_request_without_sdk_defaults(data, timeout)
+            body, options = _request_without_sdk_transform(data, timeout)
             bypass_response: Final = openai_client.post(
                 "/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
             )
@@ -1456,10 +1486,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         max_retries=None,
         organization: str | None = None,
         headers: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         try:
             openai_aclient: Final = self._get_openai_client(
                 is_async=True,
+                litellm_params=litellm_params,
                 api_key=api_key,
                 api_base=api_base,
                 timeout=timeout,
@@ -1521,6 +1553,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         aimg_generation=None,
         organization: str | None = None,
         headers: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> ImageResponse:
         data = {}
         try:
@@ -1542,10 +1575,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     max_retries=max_retries,
                     organization=organization,
                     headers=headers,
+                    litellm_params=litellm_params,
                 )
 
             openai_client: Final[OpenAI] = self._get_openai_client(
                 is_async=False,
+                litellm_params=litellm_params,
                 api_key=api_key,
                 api_base=api_base,
                 timeout=timeout,
@@ -1623,6 +1658,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         aspeech: bool | None = None,
         client=None,
         shared_session: Optional["ClientSession"] = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> HttpxBinaryResponseContent:
         if aspeech is not None and aspeech is True:
             return self.async_audio_speech(
@@ -1639,6 +1675,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 logging_obj=logging_obj,
                 client=client,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             )
 
         openai_client: Final = self._get_openai_client(
@@ -1649,6 +1686,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             max_retries=max_retries,
             client=client,
             shared_session=shared_session,
+            litellm_params=litellm_params,
         )
 
         sync_client: Final = cast(OpenAI, openai_client)
@@ -1686,6 +1724,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         logging_obj: LiteLLMLoggingObj,
         client=None,
         shared_session: Optional["ClientSession"] = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> HttpxBinaryResponseContent:
         openai_client: Final = cast(
             AsyncOpenAI,
@@ -1697,6 +1736,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 max_retries=max_retries,
                 client=client,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             ),
         )
 
@@ -1743,26 +1783,27 @@ class OpenAIFilesAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
         _is_async: bool = False,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | None:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        openai_client: OpenAI | AsyncOpenAI | None = None
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client" or k == "_is_async":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            if _is_async is True:
-                openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-            else:
-                openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        if _is_async:
+            return build_async_openai_client(
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     async def acreate_file(
         self,
@@ -1782,6 +1823,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAIFileObject | Coroutine[None, None, OpenAIFileObject]:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1791,6 +1833,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1824,6 +1867,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1833,6 +1877,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1889,6 +1934,7 @@ class OpenAIFilesAPI(BaseLLM):
         organization: str | None,
         chunk_size: int = 1024 * 1024,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> FileContentStreamingResult:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1898,6 +1944,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1952,6 +1999,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -1961,6 +2009,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -1998,6 +2047,7 @@ class OpenAIFilesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2007,6 +2057,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2047,6 +2098,7 @@ class OpenAIFilesAPI(BaseLLM):
         organization: str | None,
         purpose: str | None = None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2056,6 +2108,7 @@ class OpenAIFilesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2101,26 +2154,27 @@ class OpenAIBatchesAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
         _is_async: bool = False,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI | AsyncOpenAI | None:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        openai_client: OpenAI | AsyncOpenAI | None = None
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client" or k == "_is_async":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            if _is_async is True:
-                openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-            else:
-                openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        if _is_async:
+            return build_async_openai_client(
+                api_key=api_key,
+                api_base=api_base,
+                timeout=timeout,
+                max_retries=max_retries,
+                organization=organization,
+                litellm_params=litellm_params,
+            )
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     async def acreate_batch(
         self,
@@ -2140,6 +2194,7 @@ class OpenAIBatchesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | AsyncOpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2149,6 +2204,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2184,6 +2240,7 @@ class OpenAIBatchesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2193,6 +2250,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2227,6 +2285,7 @@ class OpenAIBatchesAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2236,6 +2295,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2276,6 +2336,7 @@ class OpenAIBatchesAPI(BaseLLM):
         after: str | None = None,
         limit: int | None = None,
         client: OpenAI | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
             api_key=api_key,
@@ -2285,6 +2346,7 @@ class OpenAIBatchesAPI(BaseLLM):
             organization=organization,
             client=client,
             _is_async=_is_async,
+            litellm_params=litellm_params,
         )
         if openai_client is None:
             raise ValueError(
@@ -2313,22 +2375,19 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: OpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAI:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            openai_client = OpenAI(**data, http_client=OpenAIHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        return build_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     def async_get_openai_client(
         self,
@@ -2338,22 +2397,19 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AsyncOpenAI:
-        received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals())
-        if client is None:
-            data: Final = {}
-            for k, v in received_args.items():
-                if k == "self" or k == "client":
-                    pass
-                elif k == "api_base" and v is not None:
-                    data["base_url"] = v
-                elif v is not None:
-                    data[k] = v
-            openai_client = AsyncOpenAI(**data, http_client=OpenAIAsyncHTTPClient())
-        else:
-            openai_client = client
-
-        return openai_client
+        if client is not None:
+            return client
+        return build_async_openai_client(
+            api_key=api_key,
+            api_base=api_base,
+            timeout=timeout,
+            max_retries=max_retries,
+            organization=organization,
+            litellm_params=litellm_params,
+        )
 
     ### ASSISTANTS ###
 
@@ -2369,6 +2425,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         limit: int | None = 20,
         before: str | None = None,
         after: str | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AsyncCursorPage[Assistant]:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2376,6 +2434,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
         request_params: Final = {
@@ -2403,6 +2462,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         aget_assistants: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, AsyncCursorPage[Assistant]]:
         ...
 
@@ -2416,6 +2477,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         aget_assistants: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> SyncCursorPage[Assistant]: 
         ...
 
@@ -2434,7 +2497,9 @@ class OpenAIAssistantsAPI(BaseLLM):
         limit: int | None = 20,
         before: str | None = None,
         after: str | None = None,
-    ):
+        *,
+        litellm_params: Mapping[str, object] | None = None,
+    ) -> Coroutine[None, None, AsyncCursorPage[Assistant]] | SyncCursorPage[Assistant]:
         if aget_assistants is not None and aget_assistants is True:
             return self.async_get_assistants(
                 api_key=api_key,
@@ -2442,6 +2507,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2450,6 +2516,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2477,6 +2544,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         create_assistant_data: dict,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Assistant:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2484,6 +2553,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2501,6 +2571,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         create_assistant_data: dict,
         client=None,
         async_create_assistants=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if async_create_assistants is not None and async_create_assistants is True:
             return self.async_create_assistants(
@@ -2509,6 +2581,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
                 create_assistant_data=create_assistant_data,
             )
@@ -2518,6 +2591,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2534,6 +2608,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         assistant_id: str,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AssistantDeleted:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2541,6 +2617,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2558,6 +2635,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         client=None,
         async_delete_assistants=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if async_delete_assistants is not None and async_delete_assistants is True:
             return self.async_delete_assistant(
@@ -2566,6 +2645,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
                 assistant_id=assistant_id,
             )
@@ -2575,6 +2655,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2593,6 +2674,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAIMessage:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2600,6 +2683,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2630,6 +2714,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         a_add_message: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, OpenAIMessage]:
         ...
 
@@ -2645,6 +2731,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         a_add_message: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> OpenAIMessage: 
         ...
 
@@ -2661,6 +2749,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client=None,
         a_add_message: bool | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if a_add_message is not None and a_add_message is True:
             return self.a_add_message(
@@ -2671,6 +2761,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2679,6 +2770,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2704,6 +2796,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> AsyncCursorPage[OpenAIMessage]:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2711,6 +2805,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2731,6 +2826,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         aget_messages: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
         ...
 
@@ -2745,6 +2842,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         aget_messages: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> SyncCursorPage[OpenAIMessage]: 
         ...
 
@@ -2760,6 +2859,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client=None,
         aget_messages=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if aget_messages is not None and aget_messages is True:
             return self.async_get_messages(
@@ -2769,6 +2870,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2777,6 +2879,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2796,6 +2899,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2803,6 +2908,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2835,6 +2941,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
         client: AsyncOpenAI | None,
         acreate_thread: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, Thread]:
         ...
 
@@ -2850,6 +2958,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
         client: OpenAI | None,
         acreate_thread: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread: 
         ...
 
@@ -2866,6 +2976,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         messages: Iterable[OpenAICreateThreadParamsMessage] | None,
         client=None,
         acreate_thread=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         """
         Here's an example:
@@ -2885,6 +2997,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
                 messages=messages,
             )
@@ -2894,6 +3007,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2921,6 +3035,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -2928,6 +3044,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -2953,6 +3070,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: AsyncOpenAI | None,
         aget_thread: Literal[True], 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, Thread]:
         ...
 
@@ -2967,6 +3086,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client: OpenAI | None,
         aget_thread: Literal[False] | None, 
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Thread: 
         ...
 
@@ -2982,6 +3103,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         organization: str | None,
         client=None,
         aget_thread=None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if aget_thread is not None and aget_thread is True:
             return self.async_get_thread(
@@ -2991,6 +3114,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -2999,6 +3123,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -3032,6 +3157,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         max_retries: int | None,
         organization: str | None,
         client: AsyncOpenAI | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Run:
         openai_client: Final = self.async_get_openai_client(
             api_key=api_key,
@@ -3039,6 +3166,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 
@@ -3143,6 +3271,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         client,
         arun_thread: Literal[True], 
         event_handler: AssistantEventHandler | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Coroutine[None, None, Run]:
         ...
 
@@ -3165,6 +3295,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         client,
         arun_thread: Literal[False] | None, 
         event_handler: AssistantEventHandler | None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> Run: 
         ...
 
@@ -3188,6 +3320,8 @@ class OpenAIAssistantsAPI(BaseLLM):
         client=None,
         arun_thread=None,
         event_handler: AssistantEventHandler | None = None,
+        *,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         if arun_thread is not None and arun_thread is True:
             if stream is not None and stream is True:
@@ -3197,6 +3331,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
+                    litellm_params=litellm_params,
                     client=client,
                 )
                 return self.async_run_thread_stream(
@@ -3224,6 +3359,7 @@ class OpenAIAssistantsAPI(BaseLLM):
                 timeout=timeout,
                 max_retries=max_retries,
                 organization=organization,
+                litellm_params=litellm_params,
                 client=client,
             )
         openai_client: Final = self.get_openai_client(
@@ -3232,6 +3368,7 @@ class OpenAIAssistantsAPI(BaseLLM):
             timeout=timeout,
             max_retries=max_retries,
             organization=organization,
+            litellm_params=litellm_params,
             client=client,
         )
 

litellm/llms/openai/responses/count_tokens/token_counter.py

@@ -27,21 +27,27 @@ from litellm.llms.openai.responses.count_tokens.transformation import (
 from litellm.llms.openai.workload_identity import (
     OpenAIWorkloadIdentityConfig,
     get_workload_identity_bearer_token,
+    openai_static_api_key,
     resolve_openai_workload_identity_config,
 )
-from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 openai_count_tokens_handler: Final = OpenAICountTokensHandler()
 
 
-def _workload_identity_config(litellm_params: Mapping[str, object]) -> OpenAIWorkloadIdentityConfig | None:
+def _workload_identity_config(
+    litellm_params: Mapping[str, object],
+) -> OpenAIWorkloadIdentityConfig | OpenAIError | None:
     api_base: Final = litellm_params.get("api_base")
-    return resolve_openai_workload_identity_config(
-        api_key=None,
-        api_base=api_base if isinstance(api_base, str) else None,
-        litellm_params=litellm_params,
-    )
+    try:
+        return resolve_openai_workload_identity_config(
+            api_key=None,
+            api_base=api_base if isinstance(api_base, str) else None,
+            litellm_params=litellm_params,
+        )
+    except OpenAIError as e:
+        return e
 
 
 class OpenAITokenCounter(BaseTokenCounter):
@@ -95,6 +101,7 @@ class OpenAITokenCounter(BaseTokenCounter):
         system: object = None,
         messages_format: TokenCountMessagesFormat = "openai",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using OpenAI's Responses API /input_tokens endpoint.
@@ -125,7 +132,18 @@ class OpenAITokenCounter(BaseTokenCounter):
         credentials: Final = get_openai_credentials(
             api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key")
         )
-        credential: Final = credentials.api_key or _workload_identity_config(litellm_params)
+        credential: Final = openai_static_api_key(credentials.api_key) or _workload_identity_config(litellm_params)
+
+        if isinstance(credential, OpenAIError):
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=credential.message,
+                status_code=credential.status_code,
+            )
 
         if credential is None:
             return TokenCountResponse(

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -55,6 +55,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
 from litellm.llms.base_llm.guardrail_translation.utils import (
     blocked_responses_stream_usage,
     effective_skip_system_message_for_guardrail,
+    list_field,
     merge_guardrailed_scoped_messages,
     role_out_of_guardrail_scope,
     scoped_structured_message_indices,
@@ -764,11 +765,11 @@ class OpenAIResponsesHandler(BaseTranslation):
             data.get("input"), data.get("instructions"), structured_messages, merged
         )
 
-    def extract_request_tool_names(self, data: dict) -> list[str]:
+    def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
         """Extract tool names from Responses API request (tools[].name for function
         and custom, tools[].server_label for mcp)."""
         names: Final[list[str]] = []
-        for tool in data.get("tools") or []:
+        for tool in list_field(data, "tools"):
             if not isinstance(tool, dict):
                 continue
             if tool.get("type") in ("function", "custom") and tool.get("name"):

litellm/llms/openai/transcriptions/handler.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final, Optional, cast
 
 import httpx
@@ -97,6 +98,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
                 max_retries=max_retries,
                 logging_obj=logging_obj,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             )
 
         openai_client: Final[OpenAI] = self._get_openai_client(
@@ -106,6 +108,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
             timeout=timeout,
             max_retries=max_retries,
             client=client,
+            litellm_params=litellm_params,
         )
 
         ## LOGGING
@@ -159,6 +162,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
         client=None,
         max_retries=None,
         shared_session: Optional["ClientSession"] = None,
+        litellm_params: Mapping[str, object] | None = None,
     ):
         try:
             openai_aclient: Final[AsyncOpenAI] = self._get_openai_client(
@@ -169,6 +173,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
                 max_retries=max_retries,
                 client=client,
                 shared_session=shared_session,
+                litellm_params=litellm_params,
             )
 
             ## LOGGING

litellm/llms/openai/vector_store_files/transformation.py

@@ -8,6 +8,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store_files.transformation import (
     BaseVectorStoreFilesConfig,
 )
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.vector_store_files import (
@@ -70,11 +71,13 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
         headers: dict[str, str],
         litellm_params: GenericLiteLLMParams | None,
     ) -> dict[str, str]:
-        litellm_params = litellm_params or GenericLiteLLMParams()
-        api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
+        )
         headers.update(
             {
-                "Authorization": f"Bearer {api_key}",
+                "Authorization": f"Bearer {bearer_token}",
                 "Content-Type": "application/json",
             }
         )

litellm/llms/openai/vector_stores/transformation.py

@@ -5,6 +5,7 @@ import httpx
 import litellm
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.vector_stores import (
@@ -48,11 +49,13 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
         }
 
     def validate_environment(self, headers: dict, litellm_params: GenericLiteLLMParams | None) -> dict:
-        litellm_params = litellm_params or GenericLiteLLMParams()
-        api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=params.api_key, api_base=params.api_base, litellm_params=params.model_dump(exclude_none=True)
+        )
         headers.update(
             {
-                "Authorization": f"Bearer {api_key}",
+                "Authorization": f"Bearer {bearer_token}",
                 "Content-Type": "application/json",
             }
         )

litellm/llms/openai/videos/transformation.py

@@ -8,11 +8,10 @@ from urllib.parse import quote
 import httpx
 from httpx._types import FileContent, FileTypes, RequestFiles
 
-import litellm
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
 from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils
-from litellm.secret_managers.main import get_secret_str
+from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
 from litellm.types.llms.openai import CreateVideoRequest
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.videos.main import (
@@ -78,16 +77,13 @@ class OpenAIVideoConfig(BaseVideoConfig):
         api_key: str | None = None,
         litellm_params: GenericLiteLLMParams | None = None,
     ) -> dict:
-        # Use api_key from litellm_params if available, otherwise fall back to other sources
-        if litellm_params and litellm_params.api_key:
-            api_key = api_key or litellm_params.api_key
-
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            }
+        params: Final = litellm_params or GenericLiteLLMParams()
+        bearer_token: Final = resolve_openai_bearer_token(
+            api_key=api_key or params.api_key,
+            api_base=params.api_base,
+            litellm_params=params.model_dump(exclude_none=True),
         )
+        headers.update({"Authorization": f"Bearer {bearer_token}"})
         return headers
 
     def get_complete_url(

litellm/llms/openai/workload_identity.py

@@ -6,10 +6,22 @@ from functools import lru_cache
 from typing import TYPE_CHECKING, Final
 from urllib.parse import urlparse
 
-import litellm
-from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
+import httpx
+from openai import DEFAULT_MAX_RETRIES as OPENAI_SDK_DEFAULT_MAX_RETRIES
+from openai import NOT_GIVEN, AsyncOpenAI, NotGiven, OpenAI
 
-from .common_utils import OpenAIError, is_openai_backed_api_base
+import litellm
+from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
+from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
+from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
+
+from .common_utils import (
+    BaseOpenAILLM,
+    OpenAIAsyncHTTPClient,
+    OpenAIError,
+    OpenAIHTTPClient,
+    is_openai_backed_api_base,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -17,6 +29,11 @@ if TYPE_CHECKING:
     from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
 
 OPENAI_WIF_CLIENT_ID: Final = "litellm"
+_IDENTITY_PARAM_ENV_NAMES: Final = (
+    ("openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"),
+    ("openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID"),
+    ("openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE"),
+)
 _SDK_UPGRADE_MESSAGE: Final = (
     "OpenAI workload identity federation requires openai>=2.32.0. "
     "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
@@ -51,17 +68,25 @@ def resolve_openai_workload_identity_config(
     )
     if static_api_key is not None:
         return None
-    effective_api_base: Final = (
-        api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
-    )
-    if not _targets_openai_api(effective_api_base):
+    return _configured_openai_workload_identity(api_base=api_base, litellm_params=litellm_params)
+
+
+def _effective_openai_api_base(api_base: str | None) -> str | None:
+    return api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
+
+
+def _configured_openai_workload_identity(
+    api_base: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> OpenAIWorkloadIdentityConfig | None:
+    if not _targets_openai_api(_effective_openai_api_base(api_base)):
         return None
-    identity_provider_id: Final = _config_value(
-        litellm_params, "openai_identity_provider_id", "OPENAI_IDENTITY_PROVIDER_ID"
+    identity_provider_id, service_account_id, token_file = (
+        _config_value(litellm_params, param_key, env_name) for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES
     )
-    service_account_id: Final = _config_value(litellm_params, "openai_service_account_id", "OPENAI_SERVICE_ACCOUNT_ID")
-    token_file: Final = _config_value(litellm_params, "openai_identity_token_file", "OPENAI_IDENTITY_TOKEN_FILE")
     if identity_provider_id is None or service_account_id is None or token_file is None:
+        if _carries_deployment_identity(litellm_params):
+            raise OpenAIError(status_code=401, message=_incomplete_identity_message(litellm_params))
         return None
     return OpenAIWorkloadIdentityConfig(
         identity_provider_id=identity_provider_id,
@@ -81,11 +106,200 @@ async def get_workload_identity_bearer_token_for_api_base(api_base: str) -> str
     return await _workload_identity_auth(config).get_token_async()
 
 
+def openai_static_api_key(api_key: str | None) -> str | None:
+    return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+
+
+def resolve_openai_bearer_token(
+    api_key: str | None,
+    api_base: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> str | None:
+    static_api_key: Final = openai_static_api_key(api_key)
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=static_api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    if workload_identity_config is None:
+        return static_api_key
+    return get_workload_identity_bearer_token(workload_identity_config)
+
+
+async def resolve_openai_bearer_token_async(
+    static_api_key: str | None,
+    api_base: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> str | None:
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=static_api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    if workload_identity_config is None:
+        return static_api_key
+    return await _workload_identity_auth(workload_identity_config).get_token_async()
+
+
+def _carries_deployment_identity(litellm_params: Mapping[str, object] | None) -> bool:
+    if litellm_params is None:
+        return False
+    return any(_param_str(litellm_params, key) is not None for key in OPENAI_WIF_KWARGS_KEYS)
+
+
+def _incomplete_identity_message(litellm_params: Mapping[str, object] | None) -> str:
+    missing: Final = ", ".join(
+        param_key
+        for param_key, env_name in _IDENTITY_PARAM_ENV_NAMES
+        if _config_value(litellm_params, param_key, env_name) is None
+    )
+    return (
+        f"OpenAI workload identity federation on this deployment is missing {missing}. "
+        "Set openai_identity_provider_id, openai_service_account_id, and openai_identity_token_file "
+        "on the deployment, or the OPENAI_IDENTITY_PROVIDER_ID / OPENAI_SERVICE_ACCOUNT_ID / "
+        "OPENAI_IDENTITY_TOKEN_FILE env vars they fall back to."
+    )
+
+
+def _param_str(litellm_params: Mapping[str, object], key: str) -> str | None:
+    value: Final = litellm_params.get(key)
+    return value if isinstance(value, str) and value else None
+
+
 def _config_value(litellm_params: Mapping[str, object] | None, param_key: str, env_name: str) -> str | None:
-    param_value: Final = litellm_params.get(param_key) if litellm_params is not None else None
-    if isinstance(param_value, str) and param_value:
-        return param_value
-    return normalize_nonempty_secret_str(get_secret_str(env_name))
+    param_value: Final = _param_str(litellm_params, param_key) if litellm_params is not None else None
+    return param_value or normalize_nonempty_secret_str(get_secret_str(env_name))
+
+
+def build_openai_client(
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    max_retries: int | None = None,
+    organization: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
+    static_key_http_client_factory: Callable[[], httpx.Client | None] | None = None,
+) -> OpenAI:
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    retries: Final = _max_retries_or_sdk_default(max_retries)
+    if workload_identity_config is None:
+        static_key_http_client: Final = (
+            None if static_key_http_client_factory is None else static_key_http_client_factory()
+        )
+        return OpenAI(
+            api_key=openai_static_api_key(api_key),
+            base_url=api_base,
+            timeout=timeout,
+            max_retries=retries,
+            organization=organization,
+            http_client=static_key_http_client or OpenAIHTTPClient(),
+        )
+    cache_params: Final = _client_cache_params(
+        is_async=False,
+        workload_identity_config=workload_identity_config,
+        api_base=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+    )
+    cached: Final = BaseOpenAILLM.get_cached_openai_client(
+        client_initialization_params=cache_params, client_type="openai"
+    )
+    if isinstance(cached, OpenAI):
+        return cached
+    process_http_client: Final = BaseOpenAILLM.get_sync_http_client()
+    client: Final = OpenAI(
+        workload_identity=workload_identity_config.to_sdk_workload_identity(),
+        base_url=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+        http_client=process_http_client,
+    )
+    BaseOpenAILLM.set_cached_openai_client(
+        openai_client=client,
+        client_type="openai",
+        client_initialization_params=cache_params,
+        litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client),
+    )
+    return client
+
+
+def build_async_openai_client(
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    max_retries: int | None = None,
+    organization: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
+    static_key_http_client_factory: Callable[[], httpx.AsyncClient | None] | None = None,
+) -> AsyncOpenAI:
+    workload_identity_config: Final = resolve_openai_workload_identity_config(
+        api_key=api_key, api_base=api_base, litellm_params=litellm_params
+    )
+    retries: Final = _max_retries_or_sdk_default(max_retries)
+    if workload_identity_config is None:
+        static_key_http_client: Final = (
+            None if static_key_http_client_factory is None else static_key_http_client_factory()
+        )
+        return AsyncOpenAI(
+            api_key=openai_static_api_key(api_key),
+            base_url=api_base,
+            timeout=timeout,
+            max_retries=retries,
+            organization=organization,
+            http_client=static_key_http_client or OpenAIAsyncHTTPClient(),
+        )
+    cache_params: Final = _client_cache_params(
+        is_async=True,
+        workload_identity_config=workload_identity_config,
+        api_base=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+    )
+    cached: Final = BaseOpenAILLM.get_cached_openai_client(
+        client_initialization_params=cache_params, client_type="openai"
+    )
+    if isinstance(cached, AsyncOpenAI):
+        return cached
+    process_http_client: Final = BaseOpenAILLM.get_async_http_client()
+    client: Final = AsyncOpenAI(
+        workload_identity=workload_identity_config.to_sdk_workload_identity(),
+        base_url=api_base,
+        timeout=timeout,
+        max_retries=retries,
+        organization=organization,
+        http_client=process_http_client,
+    )
+    BaseOpenAILLM.set_cached_openai_client(
+        openai_client=client,
+        client_type="openai",
+        client_initialization_params=cache_params,
+        litellm_owned_client=BaseOpenAILLM.owns_wrapped_http_client(process_http_client),
+    )
+    return client
+
+
+def _client_cache_params(
+    is_async: bool,
+    workload_identity_config: OpenAIWorkloadIdentityConfig,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None | NotGiven,
+    max_retries: int,
+    organization: str | None,
+) -> dict[str, object]:
+    return {
+        "api_key": None,
+        "is_async": is_async,
+        "workload_identity_config": workload_identity_config,
+        "api_base": api_base,
+        "timeout": timeout,
+        "max_retries": max_retries,
+        "organization": organization,
+    }
+
+
+def _max_retries_or_sdk_default(max_retries: int | None) -> int:
+    return OPENAI_SDK_DEFAULT_MAX_RETRIES if max_retries is None else max_retries
 
 
 def _targets_openai_api(api_base: str | None) -> bool:
@@ -95,7 +309,7 @@ def _targets_openai_api(api_base: str | None) -> bool:
     return parsed.scheme == "https" and is_openai_backed_api_base(api_base)
 
 
-@lru_cache(maxsize=16)
+@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
 def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth:
     sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth()
     return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity())

litellm/llms/openai_like/chat/handler.py

@@ -14,7 +14,7 @@ from typing_extensions import ReadOnly
 import litellm
 from litellm import LlmProviders
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
+from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.databricks.streaming_utils import ModelResponseIterator
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig

litellm/llms/openai_like/chat/transformation.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 
+from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage
 from litellm.types.utils import ModelResponse
@@ -119,6 +120,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig):
                 choice["message"] = message
 
         returned_response: Final = ModelResponse(**response_json)
+        set_provider_response_headers_in_hidden_params(returned_response, response.headers)
 
         if custom_llm_provider is not None:
             returned_response.model = custom_llm_provider + "/" + (returned_response.model or "")

litellm/llms/snowflake/chat/transformation.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict
 import httpx
 from typing_extensions import ReadOnly
 
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.prompt_templates.factory import (
     anthropic_process_openai_file_message,
     convert_to_anthropic_tool_result,
@@ -184,10 +185,11 @@ def _signed_thinking_blocks(msg: object) -> list[dict[str, object]]:  # mutable-
     blocks: Final = msg.get("thinking_blocks") if isinstance(msg, dict) else getattr(msg, "thinking_blocks", None)
     if not isinstance(blocks, list):
         return []
+    mappings: Final = tuple(filter(None, map(as_str_mapping, blocks)))
     return [
-        dict(block)
-        for block in blocks
-        if isinstance(block, Mapping) and (block.get("signature") or block.get("type") == "redacted_thinking")
+        {key: value for key, value in block.items() if key != "index"}
+        for block in mappings
+        if block.get("signature") or block.get("type") == "redacted_thinking"
     ]
 
 

litellm/llms/vertex_ai/audio_transcription/realtime_transformation.py

@@ -13,7 +13,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.llms.base_llm.realtime.transcription_protocol import (
     RealtimeTranscriptionProtocolError,
     TranscriptionAudioFormat,
-    TranscriptionSessionUpdate,
     completed_event,
     decode_pcm16_append,
     delta_event,
@@ -23,6 +22,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import (
     speech_event,
     transcription_session,
     transcription_session_created_event,
+    uses_server_vad,
 )
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
 from litellm.llms.vertex_ai.audio_transcription.transformation import (
@@ -125,7 +125,7 @@ def default_session_config(model: str) -> ChirpSessionConfig:
     )
 
 
-def parse_chirp_session_update(payload: str, expected_model: str) -> ChirpSessionConfig:
+def parse_chirp_session_update(payload: str, expected_model: str, *, drop_params: bool = False) -> ChirpSessionConfig:
     update: Final = parse_transcription_session_update(payload, ChirpProtocolError)
     if update.session_type not in (None, "transcription", "realtime"):
         raise ChirpProtocolError("Speech-to-Text streaming supports transcription sessions only")
@@ -141,7 +141,7 @@ def parse_chirp_session_update(payload: str, expected_model: str) -> ChirpSessio
         model=model,
         language=None if update.language is None else normalize_transcription_language_to_bcp47(update.language),
         sample_rate=_parse_sample_rate(update.audio_format),
-        server_vad=_parse_server_vad(update),
+        server_vad=uses_server_vad(update, "Speech-to-Text streaming", drop_params, ChirpProtocolError),
     )
 
 
@@ -161,14 +161,6 @@ def _parse_sample_rate(audio_format: TranscriptionAudioFormat | None) -> int:
     return rate
 
 
-def _parse_server_vad(update: TranscriptionSessionUpdate) -> bool:
-    if update.turn_detection_disabled:
-        return False
-    if update.turn_detection_type not in (None, "server_vad"):
-        raise ChirpProtocolError("Speech-to-Text streaming supports server_vad turn detection or null")
-    return True
-
-
 def session_created_event(config: ChirpSessionConfig, session_id: str) -> OpenAIRealtimeTranscriptionSessionCreated:
     return transcription_session_created_event(config.openai_session(session_id))
 
@@ -331,11 +323,13 @@ class VertexChirpRealtimeConfig(BaseRealtimeConfig):
         project: str,
         location: str | None,
         backend_factory: Callable[[SpeechStreamingTarget], RealtimeBackend] = _default_backend_factory,
+        drop_params: bool = False,
     ) -> None:
         self._resolve_access_token: Final = resolve_access_token
         self._project: Final = validate_vertex_transcription_project_id(project)
         self._location: Final = validate_vertex_transcription_location(location, DEFAULT_SPEECH_TO_TEXT_LOCATION)
         self._backend_factory: Final = backend_factory
+        self._drop_params: Final = drop_params
         self._transformer: Final = ChirpEventTransformer()
         self._config: ChirpSessionConfig | None = None
         self._session_id: str | None = None
@@ -423,7 +417,7 @@ class VertexChirpRealtimeConfig(BaseRealtimeConfig):
         if self._config is not None:
             verbose_logger.debug("Speech-to-Text streaming: ignoring session.update after the stream was configured")
             return ()
-        config: Final = parse_chirp_session_update(message, model)
+        config: Final = parse_chirp_session_update(message, model, drop_params=self._drop_params)
         self._config = config
         self._transformer.configure(config, self._session_id or f"sess_{uuid.uuid4().hex}")
         return (config.configure_command(),)

litellm/llms/vertex_ai/common_utils.py

@@ -23,7 +23,7 @@ from litellm.types.llms.vertex_ai import (
     PartType,
     Schema,
 )
-from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
 from litellm.utils import supports_response_schema, supports_system_messages
 
 GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType(
@@ -1274,6 +1274,7 @@ class VertexAITokenCounter(BaseTokenCounter):
         system: object | None = None,
         messages_format: TokenCountMessagesFormat = "openai",
         output_format: JsonValue = None,
+        context_management: CountContextManagement | None = None,
     ) -> TokenCountResponse | None:
         import copy
 
@@ -1352,6 +1353,7 @@ class VertexAITokenCounter(BaseTokenCounter):
         else:
             from litellm.llms.vertex_ai.count_tokens.context import (
                 VertexGeminiCountContextError,
+                count_context_edits,
                 count_context_error_response,
                 gemini_count_contents,
                 inference_request_view,
@@ -1361,7 +1363,14 @@ class VertexAITokenCounter(BaseTokenCounter):
             )
             from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
 
-            request_view: Final = inference_request_view(messages or [], system, messages_format)
+            request_view: Final = await inference_request_view(
+                messages or [],
+                system,
+                messages_format,
+                count_context_edits(context_management, tools, count_tokens_params_request, model_to_use),
+            )
+            if isinstance(request_view, VertexGeminiCountContextError):
+                return count_context_error_response(request_view, request_model, model_to_use)
             count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
             if isinstance(count_context, VertexGeminiCountContextError):
                 return count_context_error_response(count_context, request_model, model_to_use)

litellm/llms/vertex_ai/count_tokens/context.py

@@ -10,11 +10,10 @@ from typing_extensions import assert_never
 from litellm.types.llms.anthropic import AnthropicMessagesTextParam
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.llms.vertex_ai import ContentType
-from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
 
 if TYPE_CHECKING:
     from litellm.exceptions import APIError
-    from litellm.llms.anthropic.pass_through.context_management.result import PolyfillResult
 
 
 @dataclass(frozen=True, slots=True)
@@ -26,6 +25,7 @@ class VertexGeminiCountContext:
 @dataclass(frozen=True, slots=True)
 class VertexGeminiCountContextError:
     error_message: str
+    status_code: int = 400
 
 
 _SYSTEM_BLOCKS: Final = TypeAdapter(tuple[AnthropicMessagesTextParam, ...])
@@ -39,6 +39,11 @@ _VERTEX_TOOL_KEYS: Final = MappingProxyType(
 )
 
 
+_INVALID_SYSTEM: Final = VertexGeminiCountContextError(
+    error_message="Gemini system instructions only support text blocks."
+)
+
+
 def _system_parts(
     system: object | None,
 ) -> tuple[Mapping[str, JsonValue], ...] | VertexGeminiCountContextError:
@@ -49,7 +54,7 @@ def _system_parts(
     try:
         blocks: Final = _SYSTEM_BLOCKS.validate_python(system)
     except ValidationError:
-        return VertexGeminiCountContextError(error_message="Gemini system instructions only support text blocks.")
+        return _INVALID_SYSTEM
     return tuple({"text": block["text"]} for block in blocks)
 
 
@@ -157,7 +162,7 @@ def count_context_error_response(
         tokenizer_type="gemini",
         error=True,
         error_message=error.error_message,
-        status_code=400,
+        status_code=error.status_code,
     )
 
 
@@ -179,36 +184,120 @@ class InferenceRequestView:
     system: object | None
 
 
-def _client_compaction(messages: Sequence[Mapping[str, object]], system: object | None) -> "PolyfillResult | None":
+@dataclass(frozen=True, slots=True)
+class CountContextEdits:
+    spec: JsonValue
+    model: str
+    tools: tuple[Mapping[str, object], ...] | None
+    drop_invalid_edits: bool = False
+
+
+_CONTEXT_MANAGEMENT_SPEC: Final[TypeAdapter[Mapping[str, object] | tuple[Mapping[str, object], ...]]] = TypeAdapter(
+    Mapping[str, object] | tuple[Mapping[str, object], ...]
+)
+
+
+def count_context_edits(
+    context_management: CountContextManagement | None,
+    tools: Sequence[Mapping[str, object]] | None,
+    litellm_params: Mapping[str, object],
+    fallback_model: str,
+) -> CountContextEdits:
+    model: Final = litellm_params.get("model")
+    return CountContextEdits(
+        spec=None if context_management is None else context_management.spec,
+        model=model if isinstance(model, str) else fallback_model,
+        tools=None if tools is None else tuple(tools),
+        drop_invalid_edits=context_management is not None and context_management.drop_invalid_edits,
+    )
+
+
+def _typed_spec(spec: JsonValue) -> Mapping[str, object] | tuple[Mapping[str, object], ...] | None:
+    try:
+        return _CONTEXT_MANAGEMENT_SPEC.validate_python(spec)
+    except ValidationError:
+        return None
+
+
+def _typed_system(
+    system: object | None,
+) -> str | tuple[AnthropicMessagesTextParam, ...] | None | VertexGeminiCountContextError:
+    if system is None or isinstance(system, str):
+        return system
+    try:
+        return _SYSTEM_BLOCKS.validate_python(system)
+    except ValidationError:
+        return _INVALID_SYSTEM
+
+
+def _view_without_edits(
+    messages: Sequence[Mapping[str, object]],
+    system: object | None,
+    typed_system: str | Sequence[Mapping[str, object]] | None,
+) -> InferenceRequestView:
     from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
         apply_client_compaction_block_history,
     )
 
-    chat_messages: Final = [dict(message) for message in messages]
-    if system is None or isinstance(system, str):
-        return apply_client_compaction_block_history(messages=chat_messages, system=system)
-    try:
-        blocks: Final = _SYSTEM_BLOCKS.validate_python(system)
-    except ValidationError:
-        return None
-    return apply_client_compaction_block_history(messages=chat_messages, system=[dict(block) for block in blocks])
+    history: Final = apply_client_compaction_block_history(
+        messages=[dict(message) for message in messages],
+        system=typed_system
+        if typed_system is None or isinstance(typed_system, str)
+        else [dict(block) for block in typed_system],
+    )
+    if history is None:
+        return InferenceRequestView(messages=messages, system=system)
+    return InferenceRequestView(messages=history.messages, system=history.system)
 
 
-def inference_request_view(
-    messages: Sequence[Mapping[str, object]], system: object | None, messages_format: TokenCountMessagesFormat
-) -> InferenceRequestView:
-    """Apply the history rewrites /v1/messages runs before it converts a body for Gemini."""
+async def _anthropic_inference_view(
+    messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
+) -> InferenceRequestView | VertexGeminiCountContextError:
+    from litellm.llms.anthropic.pass_through.adapters.handler import prepare_context_managed_request
+    from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
     from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
 
+    typed_system: Final = _typed_system(system)
+    if isinstance(typed_system, VertexGeminiCountContextError):
+        return typed_system
+    normalized: Final = normalize_native_messages(messages)
+    spec: Final = _typed_spec(edits.spec)
+    try:
+        prepared: Final = await prepare_context_managed_request(
+            model=edits.model,
+            messages=normalized,
+            tools=None if edits.tools is None else [dict(tool) for tool in edits.tools],
+            system=[dict(block) for block in typed_system] if isinstance(typed_system, tuple) else typed_system,
+            context_management_spec=(
+                None if spec is None else dict(spec) if isinstance(spec, Mapping) else [dict(edit) for edit in spec]
+            ),
+            litellm_metadata=None,
+            additional_drop_params=None,
+            llm_router=None,
+            count_only=True,
+            drop_invalid_edits=edits.drop_invalid_edits,
+        )
+    except AnthropicContextManagementError as error:
+        if error.status_code >= 500:
+            return _view_without_edits(normalized, system, typed_system)
+        return VertexGeminiCountContextError(error_message=error.message, status_code=error.status_code)
+    if prepared is None:
+        return InferenceRequestView(messages=normalized, system=system)
+    return InferenceRequestView(messages=prepared.messages, system=prepared.system)
+
+
+async def inference_request_view(
+    messages: Sequence[Mapping[str, object]],
+    system: object | None,
+    messages_format: TokenCountMessagesFormat,
+    edits: CountContextEdits,
+) -> InferenceRequestView | VertexGeminiCountContextError:
+    """Apply the history rewrites and context edits /v1/messages runs before it converts a body for Gemini."""
     match messages_format:
         case "openai":
             return InferenceRequestView(messages=messages, system=system)
         case "anthropic":
-            normalized: Final = normalize_native_messages(messages)
-            compacted: Final = _client_compaction(normalized, system)
-            if compacted is None:
-                return InferenceRequestView(messages=normalized, system=system)
-            return InferenceRequestView(messages=compacted.messages, system=compacted.system)
+            return await _anthropic_inference_view(messages, system, edits)
         case _:
             assert_never(messages_format)
 

litellm/llms/vertex_ai/realtime/transformation.py

@@ -47,6 +47,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
     def _include_function_response_id(self) -> bool:
         return False
 
+    def _rejects_vad_tuning_with_detection_off(self) -> bool:
+        return False
+
     # ------------------------------------------------------------------
     # URL
     # ------------------------------------------------------------------
@@ -248,9 +251,12 @@ def vertex_realtime_config(
     resolve_access_token: Callable[[], Awaitable[str]],
     project: str,
     location: str | None,
+    drop_params: bool,
 ) -> VertexAIRealtimeConfig | VertexChirpRealtimeConfig:
     if is_vertex_speech_to_text_model(model):
-        return VertexChirpRealtimeConfig(resolve_access_token=resolve_access_token, project=project, location=location)
+        return VertexChirpRealtimeConfig(
+            resolve_access_token=resolve_access_token, project=project, location=location, drop_params=drop_params
+        )
     return VertexAIRealtimeConfig(
         access_token=access_token,
         project=project,

litellm/llms/xai/realtime/transformation.py

@@ -16,6 +16,7 @@ construction time (see ``handler.py``) so all normalization is isolated here
 and ``RealTimeStreaming`` stays provider-agnostic.
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 
@@ -77,6 +78,18 @@ class XAIRealtimeNormalizer:
         event = self._normalize_response_usage_event(event, event_type)
         return event
 
+    def gate_turn_detection(self, _turn_detection: object, /) -> Mapping[str, object]:
+        """xAI auto-answers under ``server_vad`` even with ``create_response`` off, so the gate is always untyped."""
+        return {"create_response": False}
+
+    @property
+    def deletes_conversation_items(self) -> bool:
+        """xAI answers ``conversation.item.delete`` with ``Item not found`` until it commits the audio item.
+
+        The guardrail blocks on an interim transcript, before that commit.
+        """
+        return False
+
     def patch_outgoing_session(self, session: dict) -> dict:
         """Patch a client ``session.update`` payload before forwarding to xAI.
 

litellm/main.py

@@ -255,6 +255,7 @@ from .llms.openai.completion.handler import OpenAITextCompletion
 from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler
 from .llms.openai.openai import OpenAIChatCompletion
 from .llms.openai.transcriptions.handler import OpenAIAudioTranscription
+from .llms.openai.workload_identity import build_openai_client
 from .llms.openai_like.chat.handler import OpenAILikeChatHandler
 from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler
 from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
@@ -1212,13 +1213,18 @@ def _register_custom_pricing_for_request(
         litellm.register_model({shared_key: entry}, persist_across_reloads=False)
         return
     litellm.register_model(
-        {
-            deployment_id: entry,
-            shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
-        },
+        {deployment_id: entry},
         persist_across_reloads=False,
         warning_display_name=shared_key,
         custom_llm_provider=custom_llm_provider,
+        add_to_provider_model_sets=False,
+    )
+    litellm.register_model(
+        {shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)},
+        persist_across_reloads=False,
+        warning_display_name=shared_key,
+        custom_llm_provider=custom_llm_provider,
+        add_to_provider_model_sets=False,
     )
 
 
@@ -5785,6 +5791,7 @@ def completion(
                     "github_copilot_user_session",
                     PROVIDER_AFFINITY_HEADER_KWARG_KEY,
                     "fireworks_forward_user_id",
+                    "responses_api_version",
                 )
                 if key in kwargs
             },
@@ -7860,17 +7867,14 @@ def adapter_completion(*, adapter_id: str, **kwargs) -> BaseModel | AdapterCompl
 
 
 def moderation(input: str, model: str | None = None, api_key: str | None = None, **kwargs) -> OpenAIModerationResponse:
-    from litellm.llms.openai.common_utils import OpenAIHTTPClient
-
-    # only supports open ai for now
-    api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
-
-    # Extract api_base from kwargs
+    static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
     api_base: Final = kwargs.get("api_base", None)
 
-    openai_client = kwargs.get("client", None)
-    if openai_client is None:
-        openai_client = openai.OpenAI(api_key=api_key, base_url=api_base, http_client=OpenAIHTTPClient())
+    openai_client: Final = kwargs.get("client", None) or build_openai_client(
+        api_key=static_api_key,
+        api_base=api_base,
+        litellm_params=kwargs,
+    )
 
     if model is not None:
         response = openai_client.moderations.create(input=input, model=model)
@@ -7893,8 +7897,7 @@ async def amoderation(
 ) -> OpenAIModerationResponse:
     from openai import AsyncOpenAI
 
-    # only supports open ai for now
-    api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+    static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
     optional_params: Final = GenericLiteLLMParams.model_validate(kwargs)
     litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
     _dynamic_api_base = None
@@ -7920,8 +7923,9 @@ async def amoderation(
         # _get_openai_client maintains in-memory caching logic for OpenAI clients
         _openai_client: AsyncOpenAI = openai_chat_completions.get_openai_client(
             is_async=True,
-            api_key=api_key,
+            api_key=static_api_key,
             api_base=optional_params.api_base or _dynamic_api_base,
+            litellm_params=optional_params.model_dump(exclude_none=True),
         )
     else:
         _openai_client = openai_client
@@ -7941,7 +7945,7 @@ async def amoderation(
         moderation_request: Final = {"input": input, "model": model}
         litellm_logging_obj.pre_call(
             input=input,
-            api_key=api_key,
+            api_key=static_api_key,
             additional_args={
                 "complete_input_dict": moderation_request,
                 "api_base": str(_openai_client.base_url),
@@ -8174,9 +8178,7 @@ def transcription(
             or get_secret("OPENAI_ORGANIZATION")
             or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
         )
-        # set API KEY
-
-        api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
+        static_api_key: Final = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
         response = openai_audio_transcriptions.audio_transcriptions(
             model=model,
             audio_file=file,
@@ -8188,7 +8190,7 @@ def transcription(
             logging_obj=litellm_logging_obj,
             max_retries=max_retries,
             api_base=api_base,
-            api_key=api_key,
+            api_key=static_api_key,
             provider_config=provider_config,
             litellm_params=litellm_params_dict,
             shared_session=shared_session,
@@ -8510,6 +8512,7 @@ def speech(
             client=client,  # pass AsyncOpenAI, OpenAI client
             aspeech=aspeech,
             shared_session=shared_session,
+            litellm_params=litellm_params_dict,
         )
     elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
         # Check if this is Azure Speech Service (Cognitive Services TTS)

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -1345,7 +1345,9 @@ class MCPRequestHandler:
         if auth is None:
             return None
         refreshed = auth.model_copy()
-        if auth.mcp_admitted_user_subject and auth.user_id:
+        if (auth.mcp_admitted_user_subject or auth.is_session_token) and auth.user_id:
+            # A session credential (``lite login`` or a redeemed ID-JAG) has no key row, its authority is
+            # sealed in the token: refresh the live user policy and keep the token's team, routes and models.
             current = await MCPRequestHandler.reload_admitted_user(auth.user_id, requires_fresh_policy=True)
             refreshed.object_permission = current.object_permission
             refreshed.object_permission_id = current.object_permission_id

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -56,6 +56,7 @@ from litellm.proxy._experimental.mcp_server.faults import (
     render_token_fault,
 )
 from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
+    JWT_BEARER_GRANT_TYPE,
     VendorCredentialState,
     aggregate_authorize,
     aggregate_token,
@@ -71,6 +72,11 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
     revoke_refresh_token,
     supported_grant_types,
 )
+from litellm.proxy._experimental.mcp_server.id_jag_grant import (
+    basic_client_id,
+    jwt_bearer_available,
+    redeem_id_jag_assertion,
+)
 from litellm.proxy._experimental.mcp_server.idp_token_exchange import (
     exchange_idp_subject_token,
     token_exchange_available,
@@ -2396,7 +2402,7 @@ async def token_endpoint(
     grant_type: str = Form(...),
     code: str = Form(None),
     redirect_uri: str = Form(None),
-    client_id: str = Form(...),
+    client_id: str | None = Form(None),
     client_secret: str | None = Form(None),
     code_verifier: str = Form(None),
     refresh_token: str | None = Form(None),
@@ -2405,6 +2411,7 @@ async def token_endpoint(
     subject_token: str | None = Form(None),
     subject_token_type: str | None = Form(None),
     requested_token_type: str | None = Form(None),
+    assertion: str | None = Form(None),
     mcp_server_name: str | None = None,
 ):
     """
@@ -2416,7 +2423,14 @@ async def token_endpoint(
     3. Return the token
     4. Return a virtual key in this response
     """
-    if mcp_server_name is None and is_gateway_dcr_client_id(client_id):
+    presented_client_id: Final = client_id or basic_client_id(request.headers.get("authorization"))
+    if presented_client_id is None:
+        raise HTTPException(
+            status_code=400, detail={"error": "invalid_request", "error_description": "client_id is required"}
+        )
+    if mcp_server_name is None and (
+        is_gateway_dcr_client_id(presented_client_id) or grant_type == JWT_BEARER_GRANT_TYPE
+    ):
         from litellm.proxy.proxy_server import (  # noqa: PLC0415  # circular import at module load
             master_key,
             user_api_key_cache,
@@ -2427,7 +2441,7 @@ async def token_endpoint(
             grant_type=grant_type,
             code=code,
             redirect_uri=redirect_uri,
-            client_id=client_id,
+            client_id=presented_client_id,
             code_verifier=code_verifier,
             refresh_token=refresh_token,
             master_key=master_key,
@@ -2439,9 +2453,12 @@ async def token_endpoint(
             subject_token_type=subject_token_type,
             requested_token_type=requested_token_type,
             exchange_subject_token=exchange_idp_subject_token,
+            assertion=assertion,
+            client_secret=client_secret,
+            redeem_assertion=redeem_id_jag_assertion,
         )
 
-    lookup_name: Final = mcp_server_name or client_id
+    lookup_name: Final = mcp_server_name or presented_client_id
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
     mcp_server = _resolve_mcp_server_by_name_or_id(lookup_name, client_ip)
     if mcp_server is None and mcp_server_name is None:
@@ -2454,7 +2471,7 @@ async def token_endpoint(
         grant_type=grant_type,
         code=code,
         redirect_uri=redirect_uri,
-        client_id=client_id,
+        client_id=presented_client_id,
         client_secret=client_secret,
         code_verifier=code_verifier,
         refresh_token=refresh_token,
@@ -2568,7 +2585,8 @@ async def native_client_auth_discovery(request: Request) -> JSONResponse:
     """The versioned contract a native client (``lite login --pkce``, or a CLI in any other
     language) reads to sign a user in through the browser and obtain a proxy credential."""
     return JSONResponse(
-        native_client_auth_contract(request, token_exchange_available()), headers=TOKEN_NO_CACHE_HEADERS
+        native_client_auth_contract(request, token_exchange_available(), jwt_bearer_available()),
+        headers=TOKEN_NO_CACHE_HEADERS,
     )
 
 
@@ -3135,7 +3153,7 @@ def _aggregate_header_scopes(request: Request) -> tuple[str, ...]:
 
 
 def _build_aggregate_authorization_server_response(
-    request: Request, token_exchange_available: bool
+    request: Request, token_exchange_available: bool, jwt_bearer_available: bool = False
 ) -> dict[str, object]:
     """RFC 8414 metadata for the gateway as the aggregate authorization server.
 
@@ -3155,9 +3173,13 @@ def _build_aggregate_authorization_server_response(
         "registration_endpoint": f"{request_base_url}/register",
         "response_types_supported": ["code"],
         "scopes_supported": [],
-        "grant_types_supported": supported_grant_types(token_exchange_available),
+        "grant_types_supported": supported_grant_types(token_exchange_available, jwt_bearer_available),
         "code_challenge_methods_supported": ["S256"],
-        "token_endpoint_auth_methods_supported": ["none", "client_secret_post"],
+        "token_endpoint_auth_methods_supported": (
+            ["none", "client_secret_post", "client_secret_basic"]
+            if jwt_bearer_available
+            else ["none", "client_secret_post"]
+        ),
     }
 
 
@@ -3193,7 +3215,7 @@ async def oauth_authorization_server_aggregate(request: Request):
     per-server row win here instead would serve an issuer of {base} against a resource that
     advertised {base}/mcp, which fails the RFC 8414 issuer check and breaks the front door.
     """
-    return _build_aggregate_authorization_server_response(request, token_exchange_available())
+    return _build_aggregate_authorization_server_response(request, token_exchange_available(), jwt_bearer_available())
 
 
 # Standard MCP pattern: /.well-known/oauth-protected-resource/mcp/{server_name}
@@ -3273,6 +3295,10 @@ def _build_oauth_authorization_server_response(
         if explicitly_named
         else request_base_url
     )
+    # With no upstream resolved the document is the gateway's own: its issuer is the bare origin
+    # and its /token serves every grant the gateway does, including the ID-JAG jwt-bearer grant.
+    gateway_owned: Final = mcp_server_name is None and issuer_path is None
+    jwt_bearer: Final = gateway_owned and jwt_bearer_available()
 
     return {
         "issuer": issuer,
@@ -3280,9 +3306,15 @@ def _build_oauth_authorization_server_response(
         "token_endpoint": token_endpoint,
         "response_types_supported": ["code"],
         "scopes_supported": (mcp_server.scopes if mcp_server and mcp_server.scopes else []),
-        "grant_types_supported": ["authorization_code", "refresh_token"],
+        "grant_types_supported": (
+            list(supported_grant_types(token_exchange_available(), jwt_bearer))
+            if gateway_owned
+            else ["authorization_code", "refresh_token"]
+        ),
         "code_challenge_methods_supported": ["S256"],
-        "token_endpoint_auth_methods_supported": ["client_secret_post"],
+        "token_endpoint_auth_methods_supported": (
+            ["none", "client_secret_post", "client_secret_basic"] if jwt_bearer else ["client_secret_post"]
+        ),
         # Claude expects a registration endpoint, even if we just fake it
         "registration_endpoint": (
             f"{request_base_url}/{mcp_server_name}/register" if mcp_server_name else f"{request_base_url}/register"