Merge remote-tracking branch 'origin/main' into litellm_warn_remaining_reasoning_drops

.github/workflows/test-linting.yml

@@ -376,10 +376,10 @@ jobs:
             exit 0
           fi
           files="$(git diff --name-only "$base" HEAD)"
-          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
+          if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
             echo "relevant=true" >> "$GITHUB_OUTPUT"
           else
-            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify."
             echo "relevant=false" >> "$GITHUB_OUTPUT"
           fi
 

.github/workflows/test-terraform-provider.yml

@@ -12,6 +12,7 @@ on:
     paths:
       - "terraform/provider/**"
       - "litellm/proxy/**"
+      - "enterprise/**"
       - ".github/workflows/test-terraform-provider.yml"
 
 permissions:

litellm/__init__.py

@@ -685,6 +685,7 @@ moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
 openinfer_models: Final[Set[str]] = set()
+kyma_models: Final[set[str]] = set()  # mutable-ok: populated when the shared model catalog loads
 streamlake_models: Final[Set[str]] = set()
 zerogpu_models: Final[Set[str]] = set()
 v0_models: Set = set()
@@ -981,6 +982,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             darkbloom_models.add(key)
         elif value.get("litellm_provider") == "openinfer":
             openinfer_models.add(key)
+        elif value.get("litellm_provider") == "kyma":
+            kyma_models.add(key)
         elif value.get("litellm_provider") == "streamlake":
             streamlake_models.add(key)
         elif value.get("litellm_provider") == "zerogpu":
@@ -1159,6 +1162,7 @@ model_list = list(
     | publicai_models
     | darkbloom_models
     | openinfer_models
+    | kyma_models
     | streamlake_models
     | zerogpu_models
     | v0_models
@@ -1290,6 +1294,7 @@ def _build_models_by_provider() -> dict:
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
         "openinfer": openinfer_models,
+        "kyma": kyma_models,
         "streamlake": streamlake_models,
         "zerogpu": zerogpu_models,
         "v0": v0_models,

litellm/_internal_context.py

@@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
 
 _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
 
+_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
+
 
 @contextmanager
 def internal_sub_call() -> Generator[None]:
@@ -123,6 +125,30 @@ def current_service_caller() -> str | None:
     return _service_caller.get()
 
 
+@contextmanager
+def mirrored_call() -> Generator[None]:
+    """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included."""
+    token: Final = _mirrored_call.set(True)
+    try:
+        yield
+    finally:
+        _restore(_mirrored_call, token)
+
+
+def in_mirrored_call() -> bool:
+    return _mirrored_call.get()
+
+
+@contextmanager
+def outside_mirrored_call() -> Generator[None]:
+    """Logging a mirrored call: the calls its callbacks start are not mirrors themselves."""
+    token: Final = _mirrored_call.set(False)
+    try:
+        yield
+    finally:
+        _restore(_mirrored_call, token)
+
+
 @contextmanager
 def pinned_billing_time(moment: datetime) -> Generator[None]:
     """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read."""

litellm/anthropic_beta_gated_fields.py

@@ -0,0 +1,325 @@
+"""Body fields that an ``anthropic-beta`` value gates.
+
+``anthropic_beta_headers_config.json`` says which beta each provider leg
+carries. When a leg drops a beta, the request fields that beta unlocks
+would make the provider 400. Under ``drop_params`` the same request degrades on
+that leg by dropping those fields too; without it the request goes out as sent
+and the provider answers its own 400, like any unsupported param. Every helper
+here is copy-on-write: the caller's containers are returned untouched when
+nothing is stripped, and a shallow copy is returned otherwise, so a fallback
+leg still sends the full request.
+"""
+
+from collections.abc import Mapping, Sequence, Set
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
+from litellm.types.llms.anthropic import (
+    ANTHROPIC_BETA_HEADER_VALUES,
+    ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
+    ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
+    ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
+    ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
+)
+
+ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
+ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
+
+THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
+THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
+SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
+SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
+SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
+SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
+
+_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
+
+BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+    {
+        ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,),
+        ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,),
+        ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,),
+        ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,),
+        ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
+        ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+        ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+    }
+)
+
+
+def _betas_unlocking(field: str) -> frozenset[str]:
+    return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields)
+
+
+_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType(
+    {
+        field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields
+    }  # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index
+)
+
+
+def field_is_gated(field: str, active_betas: Set[str]) -> bool:
+    """True when no active beta unlocks ``field``, so the field must be stripped."""
+    return _FIELD_BETAS[field].isdisjoint(active_betas)
+
+
+_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
+
+
+def _as_mapping(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MAPPING_ADAPTER.validate_python(value, strict=True)
+    except ValidationError:
+        return None
+
+
+def _as_sequence(value: object) -> tuple[object, ...] | None:
+    try:
+        return _SEQUENCE_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _is_tool_change_block(block: object) -> bool:
+    mapping: Final = _as_mapping(block)
+    block_type: Final = mapping.get("type") if mapping is not None else None
+    return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES
+
+
+def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]:
+    joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta")
+    return frozenset(piece.strip() for piece in joined.split(",") if piece.strip())
+
+
+def _validated_headers(headers: object) -> Mapping[str, str]:
+    try:
+        return _HEADERS_ADAPTER.validate_python(headers)
+    except ValidationError:
+        return MappingProxyType({})
+
+
+def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]:
+    """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched
+    case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body."""
+    body_values: Final = _as_sequence(body_betas) or ()
+    body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str))
+    return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedThinking:
+    thinking: object
+    stripped: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedMessages:
+    messages: Sequence[object]
+    stripped: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedRequest:
+    request: Mapping[str, object]
+    stripped: tuple[str, ...]
+
+
+def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking:
+    """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated."""
+    mapping: Final = _as_mapping(thinking)
+    if mapping is None or not drops_unsupported_params(drop_params):
+        return StrippedThinking(thinking=thinking, stripped=())
+    binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas)
+    display_gated: Final = mapping.get("display") == "updates" and field_is_gated(
+        THINKING_DISPLAY_UPDATES, active_betas
+    )
+    if not binding_gated and not display_gated:
+        return StrippedThinking(thinking=thinking, stripped=())
+    gated_keys: Final = frozenset(
+        {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())}
+    )
+    stripped: Final = (
+        *((THINKING_BLOCK_BINDING,) if binding_gated else ()),
+        *((THINKING_DISPLAY_UPDATES,) if display_gated else ()),
+    )
+    return StrippedThinking(
+        thinking={key: value for key, value in mapping.items() if key not in gated_keys},
+        stripped=stripped,
+    )
+
+
+def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages:
+    """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing
+    when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400."""
+    clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas)
+    output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas)
+    content: Final = message.get("content")
+    blocks: Final = _as_sequence(content) if not isinstance(content, str) else None
+    tool_changes_gated: Final = (
+        blocks is not None
+        and any(_is_tool_change_block(block) for block in blocks)
+        and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas)
+    )
+    if not clear_at_gated and not output_config_gated and not tool_changes_gated:
+        return StrippedMessages(messages=(message,), stripped=())
+    kept_content: Final[object] = (
+        [block for block in blocks if not _is_tool_change_block(block)]
+        if tool_changes_gated and blocks is not None
+        else content
+    )
+    gated_keys: Final = frozenset(
+        {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())}
+    )
+    rewritten: Final = {
+        key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys
+    }
+    stripped: Final = (
+        *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()),
+        *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()),
+        *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()),
+    )
+    if kept_content in (None, "", []) and "output_config" not in rewritten:
+        return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED))
+    return StrippedMessages(messages=(rewritten,), stripped=stripped)
+
+
+def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages:
+    mapping: Final = _as_mapping(message)
+    if mapping is None or mapping.get("role") != "system":
+        return StrippedMessages(messages=(message,), stripped=())
+    return _strip_beta_gated_system_message(mapping, active_betas)
+
+
+def _ends_on_assistant(messages: Sequence[object]) -> bool:
+    last: Final = _as_mapping(messages[-1]) if messages else None
+    return last is not None and last.get("role") == "assistant"
+
+
+def strip_beta_gated_system_messages(
+    messages: Sequence[object], active_betas: Set[str], *, drop_params: object
+) -> StrippedMessages:
+    """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same
+    sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation
+    ending on an assistant turn, which the model would read as a prefill to continue."""
+    if not drops_unsupported_params(drop_params):
+        return StrippedMessages(messages=messages, stripped=())
+    per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages)
+    stripped: Final = tuple(
+        name for result in per_message for name in result.stripped
+    )  # comprehension-ok: flattening per-message strip reports
+    if not stripped:
+        return StrippedMessages(messages=messages, stripped=())
+    kept: Final = [
+        message for result in per_message for message in result.messages
+    ]  # comprehension-ok: flattening per-message results
+    if _ends_on_assistant(kept) and not _ends_on_assistant(messages):
+        return StrippedMessages(messages=messages, stripped=())
+    return StrippedMessages(messages=kept, stripped=stripped)
+
+
+def strip_beta_gated_request_fields(
+    request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
+) -> StrippedRequest:
+    """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
+    nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
+    thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
+    messages: Final = request.get("messages")
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    system_messages: Final = (
+        strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
+        if message_sequence is not None
+        else StrippedMessages(messages=(), stripped=())
+    )
+    stripped: Final = (*thinking.stripped, *system_messages.stripped)
+    if not stripped:
+        return StrippedRequest(request=request, stripped=())
+    rewritten: Final = {
+        **request,
+        **({"thinking": thinking.thinking} if thinking.stripped else {}),
+        **({"messages": system_messages.messages} if system_messages.stripped else {}),
+    }
+    return StrippedRequest(request=rewritten, stripped=stripped)
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedNativeRequest:
+    thinking: object
+    thinking_stripped: bool
+    messages: Sequence[object]
+    messages_stripped: bool
+    stripped: tuple[str, ...]
+
+
+def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest:
+    """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists."""
+    params: Final = _as_mapping(optional_params) or MappingProxyType({})
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    return StrippedNativeRequest(
+        thinking=params.get("thinking"),
+        thinking_stripped=False,
+        messages=message_sequence if message_sequence is not None else (),
+        messages_stripped=False,
+        stripped=(),
+    )
+
+
+def strip_beta_gated_native_request(
+    headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object
+) -> StrippedNativeRequest:
+    """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta``
+    list on legs that carry betas in the body, plus the body parts it may rewrite."""
+    if not drops_unsupported_params(drop_params):
+        return unstripped_native_request(optional_params, messages)
+    active_betas: Final = active_beta_values(headers, body_betas)
+    params: Final = _as_mapping(optional_params) or MappingProxyType({})
+    thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params)
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    system_messages: Final = (
+        strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
+        if message_sequence is not None
+        else StrippedMessages(messages=(), stripped=())
+    )
+    return StrippedNativeRequest(
+        thinking=thinking.thinking,
+        thinking_stripped=bool(thinking.stripped),
+        messages=system_messages.messages,
+        messages_stripped=bool(system_messages.stripped),
+        stripped=(*thinking.stripped, *system_messages.stripped),
+    )
+
+
+def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None:
+    if not stripped:
+        return
+    described: Final = ", ".join(
+        f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})"
+        if field in _FIELD_BETAS
+        else field
+        for field in sorted(set(stripped))
+    )
+    verbose_logger.warning(
+        "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on "
+        "this request, either since this provider leg does not send that anthropic-beta (see "
+        "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user "
+        "messages.",
+        described,
+        model,
+        provider,
+    )
+
+
+def strip_beta_gated_request_body(
+    request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str
+) -> Mapping[str, object]:
+    gated: Final = strip_beta_gated_request_fields(
+        request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params
+    )
+    log_beta_gated_strips(gated.stripped, model=model, provider=provider)
+    return gated.request

litellm/anthropic_beta_headers_config.json

@@ -248,6 +248,7 @@
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
+    "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05"

litellm/anthropic_beta_headers_manager.py

@@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str:
     return aliases.get(provider, provider)
 
 
+def has_beta_headers_config(provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    return canonical_provider in config
+
+
 def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
@@ -397,7 +404,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
-) -> tuple[dict, dict]:
+) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
     Modifies both dicts in place and returns them.

litellm/caching/caching.py

@@ -455,17 +455,33 @@ class Cache:
         3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component)
         """
         litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
-        metadata_sources: Final[tuple[dict, ...]] = (
-            kwargs.get("metadata") or {},
-            kwargs.get("litellm_metadata") or {},
-            litellm_params.get("metadata") or {},
-            litellm_params.get("litellm_metadata") or {},
+        metadata_sources: Final = cast(  # cast-ok: each source is the untyped request's metadata dict or empty
+            "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]",
+            (
+                kwargs.get("metadata") or {},
+                kwargs.get("litellm_metadata") or {},
+                litellm_params.get("metadata") or {},
+                litellm_params.get("litellm_metadata") or {},
+            ),
         )
-        model_group: Final[str | None] = next(
-            (source["model_group"] for source in metadata_sources if source.get("model_group")), None
+        metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources
+        call_model_info: Final[object] = kwargs.get("model_info")
+        routed_source: Final = next(
+            (
+                source
+                for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata)
+                if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info
+            ),
+            None,
         )
+        model_group: Final = (
+            str(routed_source["model_group"])
+            if routed_source is not None
+            else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None)
+        )
+        caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources
         caching_group: Final = next(
-            (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None
+            (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None
         )
         model_id_suffix: Final = (
             f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else ""

litellm/caching/qdrant_semantic_cache.py

@@ -13,7 +13,7 @@ import asyncio
 import json
 import os
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol, cast
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from pydantic import TypeAdapter
 
@@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache):
         prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # create an embedding for prompt
-        embedding_response: Final = cast(
-            EmbeddingResponse,
-            self._get_embedding(prompt, metadata=kwargs.get("metadata")),
-        )
+        embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
 
         # get the embedding
         embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"])
@@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache):
         prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # convert to embedding
-        embedding_response: Final = cast(
-            EmbeddingResponse,
-            self._get_embedding(prompt, metadata=kwargs.get("metadata")),
-        )
+        embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
 
         # get the embedding
         embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"])

litellm/caching/redis_semantic_cache.py

@@ -14,7 +14,9 @@ import asyncio
 import json
 import os
 from collections.abc import Callable, Mapping
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
+
+from pydantic import ConfigDict, TypeAdapter
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -38,6 +40,11 @@ if TYPE_CHECKING:
     from litellm.router import Router
 
 
+_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter(
+    str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+
+
 class RedisSemanticCache(BaseCache):
     """
     Redis-backed semantic cache for LLM responses.
@@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache):
             )
         return embedding_response["data"][0]["embedding"]
 
-    def _get_cache_logic(self, cached_response: Any) -> object:
+    def _get_cache_logic(self, cached_response: str | bytes | None) -> object:
         """
         Process the cached response to prepare it for use.
 
@@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache):
             The processed cache response, or None if input was None
         """
         if cached_response is None:
-            return cached_response
+            return None
 
         # Convert bytes to string if needed
-        if isinstance(cached_response, bytes):
-            cached_response = cached_response.decode("utf-8")
+        response_text: Final = (
+            cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response
+        )
 
         # Convert string representation to Python object
         try:
-            cached_response = json.loads(cached_response)
+            return json.loads(response_text)
         except json.JSONDecodeError:
             try:
-                cached_response = ast.literal_eval(cached_response)
+                return ast.literal_eval(response_text)
             except (ValueError, SyntaxError) as e:
                 print_verbose(f"Error parsing cached response: {e}")
                 return None
 
-        return cached_response
-
     def set_cache(self, key: str, value: object, **kwargs) -> None:
         """
         Store a value in the semantic cache.
@@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache):
             similarity: Final = 1 - vector_distance
 
             cached_prompt: Final = cache_hit["prompt"]
-            cached_response: Final = cache_hit["response"]
+            cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"])
 
             # update kwargs["metadata"] with similarity, don't rewrite the original metadata
             kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
@@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache):
             similarity: Final = 1 - vector_distance
 
             cached_prompt: Final = cache_hit["prompt"]
-            cached_response: Final = cache_hit["response"]
+            cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"])
 
             # update kwargs["metadata"] with similarity, don't rewrite the original metadata
             kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity

litellm/cost_calculator.py

@@ -22,7 +22,7 @@ from litellm.constants import (
     OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
     REALTIME_METERED_USAGE_KEY,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
 from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
@@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
+from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start
 from litellm.llms.anthropic.cost_calculation import (
     cost_per_token as anthropic_cost_per_token,
 )
@@ -1396,9 +1397,9 @@ def _apply_cost_margin(
 
 def _store_cost_breakdown_in_logging_obj(
     litellm_logging_obj: LitellmLoggingObject | None,
-    prompt_tokens_cost_usd_dollar: float,
-    completion_tokens_cost_usd_dollar: float,
-    cost_for_built_in_tools_cost_usd_dollar: float,
+    prompt_tokens_cost_usd_dollar: float | None,
+    completion_tokens_cost_usd_dollar: float | None,
+    cost_for_built_in_tools_cost_usd_dollar: float | None,
     total_cost_usd_dollar: float,
     additional_costs: dict | None = None,
     original_cost: float | None = None,
@@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost(
         return float(stamped)
     if total_time:
         return total_time
-    if litellm_logging_obj is not None:
-        return litellm_logging_obj.get_response_ms()
-    return 0.0
+    if litellm_logging_obj is None:
+        return 0.0
+    reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj)
+    if reported_ms is not None and litellm_logging_obj.stream is not True:
+        return reported_ms
+    return litellm_logging_obj.get_response_ms()
+
+
+def _adjusted_provider_billed_cost(
+    reported_cost: float,
+    custom_llm_provider: str | None,
+    model: str | None,
+    region_name: str | None,
+    litellm_logging_obj: LitellmLoggingObject | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> float:
+    discounted_cost, discount_percent, discount_amount = _apply_cost_discount(
+        base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
+    )
+    adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
+        base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
+    )
+    _store_cost_breakdown_in_logging_obj(
+        litellm_logging_obj=litellm_logging_obj,
+        prompt_tokens_cost_usd_dollar=None,
+        completion_tokens_cost_usd_dollar=None,
+        cost_for_built_in_tools_cost_usd_dollar=None,
+        total_cost_usd_dollar=adjusted_cost,
+        original_cost=reported_cost,
+        discount_percent=discount_percent,
+        discount_amount=discount_amount,
+        margin_percent=margin_percent,
+        margin_fixed_amount=margin_fixed_amount,
+        margin_total_amount=margin_total_amount,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
+    return adjusted_cost
 
 
 def completion_cost(
@@ -1616,6 +1653,9 @@ def completion_cost(
             )
 
         explicit_pricing: Final = custom_pricing is True or base_model is not None
+        caller_set_prices: Final = (
+            explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None
+        )
         custom_priced_request_model: Final = model if custom_pricing is True else None
         configured_cost_per_query: Final = (
             _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None
@@ -1666,6 +1706,7 @@ def completion_cost(
             raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
         )
 
+        priced_response: Final[object] = completion_response
         for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
             model = candidate_model
             try:
@@ -1984,6 +2025,19 @@ def completion_cost(
                     total_time=total_time,
                 )
                 model_for_discount = custom_priced_request_model or model
+                if (
+                    not caller_set_prices
+                    and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None
+                ):
+                    return _adjusted_provider_billed_cost(
+                        reported_cost=reported_cost,
+                        custom_llm_provider=custom_llm_provider,
+                        model=model_for_discount,
+                        region_name=region_name if isinstance(region_name, str) else None,
+                        litellm_logging_obj=litellm_logging_obj,
+                        service_tier=service_tier,
+                        data_residency=data_residency,
+                    )
                 # Calculate cost based on prompt_tokens, completion_tokens
                 if (
                     "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"

litellm/images/main.py

@@ -641,6 +641,8 @@ def image_generation(
                     timeout=timeout,
                     client=custom_client,
                 )
+        elif custom_llm_provider == LlmProviders.KYMA.value:
+            raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
 
         return model_response
     except Exception as e:

litellm/integrations/custom_logger.py

@@ -513,7 +513,7 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         data: dict,
         user_api_key_dict: UserAPIKeyAuth,
         call_type: CallTypesLiteral,
-    ) -> Any:
+    ) -> object:
         pass
 
     async def async_post_call_streaming_hook(

litellm/integrations/datadog/datadog_cost_management.py

@@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger):
         # see _RESERVED_TAG_KEYS for the rationale.
         if self.cost_tag_keys:
             allow: Final = set(self.cost_tag_keys)
+            client_allow: Final = allow - {"is_silent_experiment"}
             for rt in log.get("request_tags") or []:
                 if not isinstance(rt, str) or ":" not in rt:
                     continue
                 k, _, v = rt.partition(":")
-                if k in allow and v:
+                if k in client_allow and v:
                     self._set_custom_tag(tags, k, v)
             for k, v in metadata.items():
                 if k in allow and v is not None and not isinstance(v, (dict, list)):
@@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
                 nested = metadata.get(nested_key)
                 if isinstance(nested, dict):
                     for k, v in nested.items():
-                        if k in allow and v is not None and not isinstance(v, (dict, list)):
+                        if k in client_allow and v is not None and not isinstance(v, (dict, list)):
                             self._set_custom_tag(tags, k, str(v))
 
         return tags

litellm/integrations/langfuse/langfuse.py

@@ -833,6 +833,12 @@ class LangFuseLogger:
                 ("vertex_location", vertex_location, bool(vertex_location)),
                 ("aws_region_name", aws_region_name, bool(aws_region_name)),
                 ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs),
+                (
+                    "is_silent_experiment",
+                    "is_silent_experiment" in allowlisted_metadata
+                    and allowlisted_metadata["is_silent_experiment"] is True,
+                    "is_silent_experiment" in allowlisted_metadata,
+                ),
             )
             enrichments: Final[Mapping[str, object]] = {
                 key: value for key, value, include in candidate_enrichments if include

litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py

@@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC):
         pass
 
 
-def cast_as_primitive_value_type(value) -> str | bool | int | float:
+def cast_as_primitive_value_type(value: object) -> str | bool | int | float:
     """
     Converts a value to an OTEL-supported primitive for Arize/Phoenix observability.
     """

litellm/integrations/prometheus.py

@@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload(
         **(requester_metadata if isinstance(requester_metadata, dict) else {}),
         **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}),
         **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
+        "is_silent_experiment": (
+            "true"
+            if "is_silent_experiment" in standard_logging_metadata
+            and standard_logging_metadata["is_silent_experiment"] is True
+            else None
+        ),
     }
 
 

litellm/interactions/litellm_responses_transformation/handler.py

@@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler:
         return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
             responses_response=responses_api_response,
             model=model,
+            custom_llm_provider=custom_llm_provider,
         )
 
     async def async_interactions_api_handler(
@@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler:
         return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
             responses_response=responses_api_response,
             model=model,
+            custom_llm_provider=responses_request.get("custom_llm_provider"),
         )

litellm/interactions/litellm_responses_transformation/transformation.py

@@ -12,6 +12,7 @@ from typing import Final, cast
 
 from pydantic import BaseModel
 
+from litellm.litellm_core_utils.core_helpers import billed_usage_cost
 from litellm.types.interactions import (
     InteractionInput,
     InteractionsAPIOptionalRequestParams,
@@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig:
     def transform_responses_response_to_interactions_response(
         responses_response: ResponsesAPIResponse,
         model: str | None = None,
+        custom_llm_provider: str | None = None,
     ) -> InteractionsAPIResponse:
         """
         Transform a Responses API response to an Interactions API response.
@@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig:
         # (total_input_tokens, total_output_tokens)
         usage: Final[object] = getattr(responses_response, "usage", None)
         if usage:
+            reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider)
             interactions_response_dict["usage"] = {
                 "total_input_tokens": getattr(usage, "input_tokens", 0),
                 "total_output_tokens": getattr(usage, "output_tokens", 0),
+                **({} if reported_cost is None else {"cost": reported_cost}),
             }
 
         # Add updated (same as created for now)

litellm/litellm_core_utils/core_helpers.py

@@ -2,6 +2,7 @@
 ## Helper utilities
 import copy
 import logging
+import math
 import re
 from collections.abc import Collection, Iterable, Iterator, Mapping
 from types import MappingProxyType
@@ -1016,6 +1017,16 @@ def chunk_usage(chunk: object) -> object:
     return getattr(chunk, "usage", None)
 
 
+USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
+
+
+def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
+    if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
+        return None
+    cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
+    return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
+
+
 _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
 
 

litellm/litellm_core_utils/litellm_logging.py

@@ -23,7 +23,7 @@ from httpx import Response, Timeout
 from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
-from litellm._internal_context import post_response_phase
+from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase
 from litellm._logging import (
     is_debugging_on,
     redact_string,
@@ -162,7 +162,7 @@ from litellm.types.llms.openai import (
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse
-from litellm.types.router import RouterRateLimitError
+from litellm.types.router import RouterRateLimitError, SilentExperimentFlag
 from litellm.types.utils import (
     DEPLOYMENT_SCOPED_PRICING_FIELDS,
     AgenticLoopParams,
@@ -315,7 +315,7 @@ else:
 _in_memory_loggers: Final[list[CustomLogger]] = []
 
 _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
-    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
+    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment")
 )
 _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
     frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
@@ -809,6 +809,7 @@ class Logging(LiteLLMLoggingBaseClass):
             litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
 
         self.litellm_params = litellm_params
+        self.is_silent_experiment: Final = in_mirrored_call()
 
         # Initialize cost breakdown field
         self.cost_breakdown: CostBreakdown | None = None
@@ -1941,10 +1942,10 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def set_cost_breakdown(
         self,
-        input_cost: float,
-        output_cost: float,
+        input_cost: float | None,
+        output_cost: float | None,
         total_cost: float,
-        cost_for_built_in_tools_cost_usd_dollar: float,
+        cost_for_built_in_tools_cost_usd_dollar: float | None,
         additional_costs: dict | None = None,
         original_cost: float | None = None,
         discount_percent: float | None = None,
@@ -1983,14 +1984,17 @@ class Logging(LiteLLMLoggingBaseClass):
 
         self.billed_token_rates = billed_token_rates
         self.cost_breakdown = CostBreakdown(
-            input_cost=input_cost,
-            output_cost=output_cost,
             total_cost=total_cost,
-            tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
             service_tier=service_tier,
             data_residency=data_residency,
             vertex_location=vertex_location,
         )
+        if input_cost is not None:
+            self.cost_breakdown["input_cost"] = input_cost
+        if output_cost is not None:
+            self.cost_breakdown["output_cost"] = output_cost
+        if cost_for_built_in_tools_cost_usd_dollar is not None:
+            self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar
         if cache_read_cost is not None and cache_read_cost > 0:
             self.cost_breakdown["cache_read_cost"] = cache_read_cost
         if cache_creation_cost is not None and cache_creation_cost > 0:
@@ -3253,7 +3257,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """Restores trace_id/session_id contextvars once this attempt's own success
         logging (including any nested calls its callbacks trigger) is fully done."""
         try:
-            with post_response_phase():
+            with post_response_phase(), outside_mirrored_call():
                 return self._success_handler_body(
                     result=result,
                     start_time=start_time,
@@ -3753,7 +3757,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """Restores trace_id/session_id contextvars once this attempt's own success
         logging (including any nested calls its callbacks trigger) is fully done."""
         try:
-            with post_response_phase():
+            with post_response_phase(), outside_mirrored_call():
                 return await self._async_success_handler_body(
                     result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs
                 )
@@ -4270,13 +4274,14 @@ class Logging(LiteLLMLoggingBaseClass):
         ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a
         CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook."""
         try:
-            return self._failure_handler_body(
-                exception=exception,
-                traceback_exception=traceback_exception,
-                start_time=start_time,
-                end_time=end_time,
-                for_async_request=for_async_request,
-            )
+            with outside_mirrored_call():
+                return self._failure_handler_body(
+                    exception=exception,
+                    traceback_exception=traceback_exception,
+                    start_time=start_time,
+                    end_time=end_time,
+                    for_async_request=for_async_request,
+                )
         finally:
             self._restore_correlation_context()
 
@@ -4469,12 +4474,13 @@ class Logging(LiteLLMLoggingBaseClass):
         """Restores trace_id/session_id contextvars once this attempt's own failure
         logging (including any nested calls its callbacks trigger) is fully done."""
         try:
-            return await self._async_failure_handler_body(
-                exception=exception,
-                traceback_exception=traceback_exception,
-                start_time=start_time,
-                end_time=end_time,
-            )
+            with outside_mirrored_call():
+                return await self._async_failure_handler_body(
+                    exception=exception,
+                    traceback_exception=traceback_exception,
+                    start_time=start_time,
+                    end_time=end_time,
+                )
         finally:
             self._restore_correlation_context()
 
@@ -6563,6 +6569,7 @@ class StandardLoggingPayloadSetup:
         start_time: dt_object | None = None,
         response_id: str | None = None,
         custom_llm_provider: str | None = None,
+        is_silent_experiment: bool = False,
     ) -> StandardLoggingMetadata:
         """
         Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata.
@@ -6636,6 +6643,7 @@ class StandardLoggingPayloadSetup:
                 proxy_stamped_used_client_oauth_token(metadata, litellm_params),
                 custom_llm_provider,
             ),
+            **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
         )
         if isinstance(metadata, dict):
             for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
@@ -7436,6 +7444,7 @@ def get_standard_logging_object_payload(
             start_time=start_time,
             response_id=id,
             custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None,
+            is_silent_experiment=logging_obj.is_silent_experiment is True,
         )
         _request_body: Final = proxy_server_request.get("body", {})
         end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get(
@@ -7720,9 +7729,8 @@ def get_standard_logging_metadata(
     )
     if isinstance(metadata, dict):
         # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
-        for key in StandardLoggingMetadata.__annotations__:
-            if key in metadata:
-                clean_metadata[key] = metadata[key]
+        for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}:
+            clean_metadata[key] = metadata[key]
 
         if metadata.get("user_api_key") is not None:
             if is_valid_sha256_hash(str(metadata.get("user_api_key"))):

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -27,6 +27,15 @@ def _timing_window_start(
     return start_time, False
 
 
+def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None:
+    reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms")
+    if reported_ms is None:
+        return None
+    logging_start: Final = logging_obj.start_time
+    window_start: Final = _timing_window_start(logging_start, logging_obj)[0]
+    return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000
+
+
 def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None:
     if not isinstance(windows, (list, tuple)):
         return None

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -12,7 +12,11 @@ from typing_extensions import ReadOnly, Required
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
+from litellm.litellm_core_utils.core_helpers import (
+    USAGE_COST_BILLED_PROVIDERS,
+    chunk_usage,
+    provider_reported_usage_cost,
+)
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
@@ -1352,7 +1356,7 @@ def stream_chunk_builder_text_completion(
     return TextCompletionResponse(**response)
 
 
-_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
+_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
 
 
 def _logged_provider(logging_obj: Optional["Logging"]) -> object:

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation):
         message, _ = serialize_http_exception_detail(exc.detail)
         return tuple(anthropic_sse_error_frames(message))
 
-    def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
+    def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse":
         import uuid
 
-        from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
-            FakeAnthropicMessagesStreamIterator,
-        )
         from litellm.llms.base_llm.guardrail_translation.utils import (
             blocked_response_usage,
         )
         from litellm.types.utils import AnthropicMessagesResponse
 
-        block_response: Final = AnthropicMessagesResponse(
+        return AnthropicMessagesResponse(
             id=f"msg_{uuid.uuid4()}",
             type="message",
             role="assistant",
             content=[{"type": "text", "text": exc.message}],
             model=exc.model,
             stop_reason="end_turn",
-            usage=blocked_response_usage(getattr(exc, "original_response", None)),
+            usage=blocked_response_usage(exc.original_response),
         )
-        return list(FakeAnthropicMessagesStreamIterator(response=block_response))
+
+    def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
+        from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
+            FakeAnthropicMessagesStreamIterator,
+        )
+
+        return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc)))
 
     def _block_continuation_chunks(
         self, exc: "ModifyResponseException", responses_so_far: Sequence[object]

litellm/llms/anthropic/chat/handler.py

@@ -17,6 +17,7 @@ import litellm
 import litellm.litellm_core_utils
 import litellm.types
 import litellm.types.utils
+from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.anthropic_beta_headers_manager import (
     update_request_with_filtered_beta,
 )
@@ -506,11 +507,20 @@ class AnthropicChatCompletion(BaseLLM):
             The pair stays mutable because the streaming path rewrites it in
             place (`data["stream"] = True`) before sending.
             """
-            request_headers, data = update_request_with_filtered_beta(
+            request_headers, filtered_data = update_request_with_filtered_beta(
                 headers=headers,
                 request_data=request_data,
                 provider=custom_llm_provider,
             )
+            data: Final = dict(
+                strip_beta_gated_request_body(
+                    filtered_data,
+                    request_headers,
+                    drop_params=litellm_params.get("drop_params"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # litellm_params is the handler's bare dict
+                    model=model,
+                    provider=custom_llm_provider,
+                )
+            )
             logging_obj.pre_call(
                 input=messages,
                 api_key=api_key,

litellm/llms/anthropic/common_utils.py

@@ -51,6 +51,7 @@ from litellm.types.llms.anthropic import (
     ANTHROPIC_OAUTH_BETA_HEADER,
     ANTHROPIC_OAUTH_TOKEN_PREFIX,
     ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
+    ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
     ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
     ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
     AllAnthropicToolsValues,
@@ -600,6 +601,20 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             return False
         return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
 
+    def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]:
+        return (
+            *(
+                (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)
+                if self.is_thinking_display_updates_used(thinking)
+                else ()
+            ),
+            *(
+                (ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,)
+                if isinstance(thinking, dict) and "block_binding" in thinking
+                else ()
+            ),
+        )
+
     def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool:
         for message in messages:
             if message_field(message, "role") != "system":
@@ -1197,7 +1212,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         *,
         custom_llm_provider: str,
         is_mid_conversation_output_config_used: bool = False,
-        is_thinking_display_updates_used: bool = False,
+        thinking: AnthropicThinkingParam | None = None,
         is_mid_conversation_tool_change_used: bool = False,
     ) -> list[str]:
         """
@@ -1234,13 +1249,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         if is_mid_conversation_output_config_used:
             betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER)
 
-        thinking_display_betas: Final = (
-            (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else ()
-        )
         tool_change_betas: Final = (
             (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else ()
         )
-        return list(set(betas).union(thinking_display_betas, tool_change_betas))
+        return list(set(betas).union(self.thinking_betas(thinking), tool_change_betas))
 
     @staticmethod
     def _make_api_key_auth_header(
@@ -1302,7 +1314,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         api_base: str | None = None,
         use_bearer_for_custom_base: bool = False,
         is_mid_conversation_output_config_used: bool = False,
-        is_thinking_display_updates_used: bool = False,
+        thinking: AnthropicThinkingParam | None = None,
         is_mid_conversation_tool_change_used: bool = False,
         wif_minted: bool = False,
     ) -> dict:
@@ -1363,7 +1375,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             betas.update(user_anthropic_beta_headers)
 
         all_betas: Final = betas.union(
-            (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (),
+            self.thinking_betas(thinking),
             (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (),
         )
 
@@ -1450,7 +1462,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             auth_token=auth_token,
             file_id_used=file_id_used,
             is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
-            is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")),
+            thinking=optional_params.get("thinking"),
             is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages),
             web_search_tool_used=web_search_tool_used,
             is_vertex_request=optional_params.get("is_vertex_request", False),

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -204,7 +204,7 @@ from litellm.types.llms.openai import (
     ChatCompletionUserMessage,
     ToolMessageContentPart,
 )
-from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
+from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage
 from litellm.utils import supports_mid_conversation_system
 
 from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
@@ -364,6 +364,9 @@ class AnthropicAdapter:
 _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
 
 
+_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value})
+
+
 class LiteLLMAnthropicMessagesAdapter:
     def __init__(self):
         pass
@@ -1135,7 +1138,8 @@ class LiteLLMAnthropicMessagesAdapter:
         """Translate Anthropic thinking to either thinking or reasoning_effort.
 
         A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one
-        speaks that param. Carrying its adaptive effort tier alongside takes two different params,
+        speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only
+        ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params,
         because the two are not interchangeable at the provider mapping below.
 
         Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking``
@@ -1166,7 +1170,9 @@ class LiteLLMAnthropicMessagesAdapter:
         is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(
             model
         )
-        is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
+        is_claude_target: Final = (
+            self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
+        ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS
         output_config: Final = anthropic_message_request.get("output_config")
 
         if is_claude_target:

litellm/llms/anthropic/messages/native/transformation.py

@@ -2,10 +2,21 @@ from collections.abc import AsyncIterator, Mapping, Sequence
 from typing import ClassVar, Final
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config
+from litellm.anthropic_beta_gated_fields import (
+    log_beta_gated_strips,
+    strip_beta_gated_native_request,
+    strip_beta_gated_system_messages,
+    unstripped_native_request,
+)
+from litellm.anthropic_beta_headers_manager import (
+    get_unsupported_headers,
+    has_beta_headers_config,
+    is_beta_header_supported_by_bundled_config,
+)
 from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
@@ -20,7 +31,6 @@ from litellm.types.llms.anthropic import (
     ANTHROPIC_ADVISOR_TOOL_TYPE,
     ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
-    ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
     AnthropicMessagesRequest,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
@@ -51,6 +61,8 @@ DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
 
 _CALLER_CREDENTIAL_HEADERS: Final = frozenset({"x-api-key", "authorization"})
 
+_MESSAGE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
+
 
 def _carries_caller_credential(headers: Mapping[str, str]) -> bool:
     """Whether the caller sent their own Anthropic credential, in which case this passthrough
@@ -85,6 +97,9 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa
     return response
 
 
+_MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
+
+
 class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     _workload_identity_eligible: ClassVar[bool] = True
     _report_reasoning_drops: ClassVar[bool] = True
@@ -230,7 +245,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         else:
             return system_param
 
-    def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None:
+    def _strip_beta_gated_system_turns(
+        self, messages: Sequence[Mapping[str, object]], model: str, drop_params: object
+    ) -> Sequence[Mapping[str, object]]:
+        if not has_beta_headers_config(self._resolved_provider):
+            return messages
+        gated: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params)
+        log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider)
+        return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else messages
+
+    def _normalize_system_role_messages(
+        self, anthropic_messages_request: dict, model: str, drop_params: object
+    ) -> None:
         """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic
         ``/v1/messages`` contract, which the first-party API, Bedrock Invoke,
         Vertex, and Azure Foundry all enforce identically.
@@ -278,7 +304,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                 custom_llm_provider=self.custom_llm_provider,
                 key="supports_mid_conversation_system",
             )
-            else list(convert_mid_conversation_system_turns(messages[leading_count:]))
+            else list(
+                convert_mid_conversation_system_turns(
+                    self._strip_beta_gated_system_turns(
+                        messages[leading_count:],  # pyright: ignore[reportUnknownArgumentType]  # messages comes from the bare request dict
+                        model=model,
+                        drop_params=drop_params,
+                    )
+                )
+            )
         )
         if hoisted or remaining != messages:
             anthropic_messages_request["messages"] = remaining
@@ -337,7 +371,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                     model=model,
                 ),
             )
-        return self._finalize_messages_headers(headers, optional_params, messages), api_base
+        return self.finalize_messages_headers(headers, optional_params, messages), api_base
 
     async def avalidate_anthropic_messages_environment(
         self,
@@ -377,7 +411,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                     model=model,
                 ),
             )
-        return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base
+        return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base
 
     def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]:
         if auth_header is None:
@@ -411,7 +445,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
 
         return config_allows_workload_identity(self)
 
-    def _finalize_messages_headers(
+    def finalize_messages_headers(
         self,
         headers: dict,  # mutable-ok: out-param
         optional_params: dict,  # mutable-ok: out-param
@@ -620,6 +654,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         if thinking_enabled or effort_enabled:
             optional_params.pop("temperature", None)
 
+    def strips_beta_gated_fields_from_final_headers(self) -> bool:
+        """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke),
+        which strip beta-gated fields themselves once that list exists, and on providers with no row in
+        ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail."""
+        return has_beta_headers_config(self._resolved_provider)
+
     def transform_anthropic_messages_request(
         self,
         model: str,
@@ -723,18 +763,37 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         if not _has_advisor:
             messages = strip_advisor_blocks_from_messages(messages)
 
+        gated: Final = (
+            strip_beta_gated_native_request(
+                headers,  # pyright: ignore[reportUnknownArgumentType]  # the base signature types this as a bare dict
+                anthropic_messages_optional_request_params,  # pyright: ignore[reportUnknownArgumentType]  # the base signature types this as a bare dict
+                messages,  # pyright: ignore[reportUnknownArgumentType]  # the base signature types this as a bare list
+                drop_params=getattr(litellm_params, "drop_params", None),
+            )
+            if self.strips_beta_gated_fields_from_final_headers()
+            else unstripped_native_request(anthropic_messages_optional_request_params, messages)  # pyright: ignore[reportUnknownArgumentType]  # the base signature types these as bare dicts
+        )
+        log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider)
         anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
             messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
-                drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None))
+                drop_non_string_tool_result_texts(
+                    _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages,
+                    getattr(litellm_params, "drop_params", None),
+                )
             ),
             max_tokens=max_tokens,
             model=model,
             **anthropic_messages_optional_request_params,
         )
+        request: Final = (
+            {**dict(anthropic_messages_request), "thinking": gated.thinking}
+            if gated.thinking_stripped
+            else dict(anthropic_messages_request)  # pyright: ignore[reportUnknownVariableType]  # built from the base signature's bare dict
+        )
         if not isinstance(tools, list):
-            return dict(anthropic_messages_request)
+            return request
         return {
-            **anthropic_messages_request,
+            **request,
             "tools": self.tools_for_upstream(
                 OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params
             ),
@@ -869,17 +928,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         if AnthropicModelInfo().is_tool_search_used(tools):
             beta_values.add(get_tool_search_beta_header(custom_llm_provider))
 
-        thinking_display_betas: Final = (
-            (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)
-            if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking"))
-            else ()
-        )
+        thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking"))
         tool_change_betas: Final = (
             (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
             if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages)
             else ()
         )
-        all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas)
+        all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas)
         dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
         forwarded_beta_values: Final = (  # pyright: ignore[reportUnknownVariableType]  # derived from the bare beta set
             all_beta_values - {dangerous_tool_use}

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -665,12 +665,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
             TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
             dict(options),
         )
-        body: Final = _BaselineAnthropicMessagesConfig().transform_anthropic_messages_request(
+        config: Final = _BaselineAnthropicMessagesConfig()
+        body: Final = config.transform_anthropic_messages_request(
             model=resolved_model,
             messages=strip_provider_specific_fields_from_anthropic_messages(messages),
             anthropic_messages_optional_request_params=filtered,
             litellm_params=GenericLiteLLMParams(),
-            headers={},
+            headers=config.finalize_messages_headers({}, dict(filtered), list(messages)),  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes and returns bare dicts
         )
         return MappingProxyType(_JSON_OBJECT.validate_python(body))
     except Exception:  # noqa: BLE001  # an unsupported hypothetical request is unavailable, never an inference failure

litellm/llms/azure_ai/anthropic/handler.py

@@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen
 
 import copy
 import json
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
 
 import httpx
 
+from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
 from litellm.llms.anthropic.chat.transformation import JsonModeRequest
@@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
             litellm_params=litellm_params,
         )
 
-        data: Final = config.transform_request(
+        request_body: Final[Mapping[str, object]] = config.transform_request(
             model=model,
             messages=messages,
             optional_params=optional_params,
             litellm_params=litellm_params,
             headers=headers,
         )
+        data: Final = dict(
+            strip_beta_gated_request_body(
+                request_body,
+                headers,  # pyright: ignore[reportUnknownArgumentType]  # validate_environment returns a bare dict
+                drop_params=litellm_params.get("drop_params"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # litellm_params is the handler's bare dict
+                model=model,
+                provider=custom_llm_provider,
+            )
+        )
 
         ## LOGGING
         logging_obj.pre_call(

litellm/llms/azure_ai/anthropic/messages_transformation.py

@@ -145,6 +145,8 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
             litellm_params=litellm_params,
             headers=headers,
         )
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)
+        self._normalize_system_role_messages(
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
         self._remove_scope_from_cache_control(anthropic_messages_request)
         return anthropic_messages_request

litellm/llms/azure_ai/anthropic/transformation.py

@@ -107,9 +107,11 @@ class AzureAnthropicConfig(AnthropicConfig):
             is_vertex_request=optional_params.get("is_vertex_request", False),
             user_anthropic_beta_headers=user_anthropic_beta_headers,
             mcp_server_used=mcp_server_used,
+            thinking=optional_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # optional_params is the base config's bare dict
         )
         # Merge headers - Azure auth (api-key or Authorization) takes precedence
-        headers = {**anthropic_headers, **headers}
+        merged_beta_header: Final = {k: v for k, v in anthropic_headers.items() if k == "anthropic-beta"}
+        headers = {**anthropic_headers, **headers, **merged_beta_header}
 
         # Ensure anthropic-version header is set
         if "anthropic-version" not in headers:

litellm/llms/azure_ai/passthrough/transformation.py

@@ -5,8 +5,9 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 
+from litellm._lazy_imports import get_default_encoding
 from litellm._logging import verbose_logger
 from litellm.llms.azure_ai.common_utils import (
     AzureFoundryModelInfo,
@@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import (
     relayed_body,
     strip_leading_model_segment,
 )
+from litellm.llms.cohere.embed.transformation import (
+    CohereEmbeddingRowShape,
+    CohereImageTokenBilling,
+    cohere_embedding_response,
+)
 from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.rerank import RerankResponse
-from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject
+from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject
 
 if TYPE_CHECKING:
     from httpx import URL, Response
@@ -35,6 +41,7 @@ if TYPE_CHECKING:
 
 
 EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({})
+ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True))
 
 
 def api_version_from(litellm_params: Mapping[str, object]) -> str | None:
@@ -85,9 +92,29 @@ def relay_query_params(
     return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version})
 
 
+def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]:
+    try:
+        return ECHOED_TEXTS.validate_python(body.get("texts"))
+    except ValidationError:
+        return ()
+
+
+def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse:
+    return cohere_embedding_response(
+        body,
+        None,
+        get_default_encoding(),
+        echoed_texts(body),
+        CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS,
+        CohereEmbeddingRowShape.TYPED,
+    )
+
+
 FOUNDRY_RELAY_SHAPES: Final = (
     RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate),
     RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate),
+    RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response),
+    RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response),
 )
 
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -17,6 +17,8 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking
+from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.constants import (
     BEDROCK_MIN_THINKING_BUDGET_TOKENS,
     RESPONSE_FORMAT_TOOL_NAME,
@@ -1828,7 +1830,16 @@ class AmazonConverseConfig(BaseConfig):
                 filtered_tools.append(tool)
 
         base_model: Final = BedrockModelInfo.get_base_model(model)
-        client_beta_list: Final = get_anthropic_beta_from_headers(headers or {})
+        header_beta_list: Final = get_anthropic_beta_from_headers(headers or {})
+        client_beta_list: Final = header_beta_list or filter_and_transform_beta_headers(
+            sorted(
+                active_beta_values(
+                    {},
+                    additional_request_params.get("anthropic_beta"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # additional_request_params is a bare dict
+                )
+            ),
+            provider="bedrock_converse",
+        )
         eager_beta: Final = (
             (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,)
             if base_model.startswith("anthropic")
@@ -1836,7 +1847,15 @@ class AmazonConverseConfig(BaseConfig):
             and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list
             else ()
         )
-        anthropic_beta_list: Final = [*client_beta_list, *eager_beta]
+        thinking_betas: Final = AnthropicModelInfo().thinking_betas(
+            additional_request_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # additional_request_params is a bare dict
+        )
+        missing_thinking_betas: Final = (
+            tuple(beta for beta in thinking_betas if beta not in client_beta_list)
+            if base_model.startswith("anthropic")
+            else ()
+        )
+        anthropic_beta_list: Final = [*client_beta_list, *eager_beta, *missing_thinking_betas]
 
         # Only separate tools if computer use tools are actually present
         if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
@@ -1906,7 +1925,8 @@ class AmazonConverseConfig(BaseConfig):
                 else:
                     computer_use_header = "computer-use-2024-10-22"
 
-                anthropic_beta_list.append(computer_use_header)
+                if computer_use_header not in anthropic_beta_list:
+                    anthropic_beta_list.append(computer_use_header)
                 # Transform computer use tools to proper Bedrock format
                 transformed_computer_tools: Final = self._transform_computer_use_tools(computer_use_tools)
                 additional_request_params["tools"] = transformed_computer_tools
@@ -2031,6 +2051,16 @@ class AmazonConverseConfig(BaseConfig):
         bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
             original_tools, model, headers, additional_request_params, litellm_params, deployment_model
         )
+        requested_thinking: Final = additional_request_params.get("thinking")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # _prepare_request_params hands back a bare dict
+        body_betas: Final = additional_request_params.get("anthropic_beta")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # same bare dict
+        gated_thinking: Final = strip_beta_gated_thinking(
+            requested_thinking,  # pyright: ignore[reportUnknownArgumentType]  # same bare dict
+            active_beta_values({}, body_betas) | frozenset(anthropic_beta_list),  # pyright: ignore[reportUnknownArgumentType]  # _process_tools_and_beta hands back a bare list
+            drop_params=drop_params,
+        )
+        if gated_thinking.stripped:
+            log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse")
+            additional_request_params["thinking"] = gated_thinking.thinking
 
         # Append cachePoint to tools if cache_control_injection_points has tool_config
         cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None)

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
 import httpx
 from pydantic import TypeAdapter
 
+from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         if beta_list:
             _anthropic_request["anthropic_beta"] = beta_list
 
-        return _anthropic_request
+        return dict(
+            strip_beta_gated_request_body(
+                _anthropic_request,  # pyright: ignore[reportUnknownArgumentType]  # built by the base signature's bare dict transform
+                {},
+                drop_params=litellm_params.get("drop_params"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # litellm_params is the base signature's bare dict
+                model=model,
+                provider="bedrock_invoke",
+            )
+        )
 
     @property
     def uses_async_transform_request(self) -> bool:
@@ -294,7 +303,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools)
         input_examples_used: Final = self.is_input_examples_used(tools)
         is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
-        is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking)
 
         client_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
         user_beta_set: Final = (
@@ -312,7 +320,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
             mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
             custom_llm_provider="bedrock",
             is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
-            is_thinking_display_updates_used=is_thinking_display_updates_used,
+            thinking=thinking,
         )
         beta_set.update(auto_betas)
 

litellm/llms/bedrock/claude_platform/transformation.py

@@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
             ),
             code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")),
             container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params),
+            thinking=optional_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # optional_params is the base config's bare dict
         )
         anthropic_headers["anthropic-workspace-id"] = workspace_id
         return {**headers, **anthropic_headers}

litellm/llms/bedrock/embed/embedding.py

@@ -9,9 +9,11 @@ from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Final, get_args, overload
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
+from litellm.litellm_core_utils.hidden_params import set_hidden_param
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
 from litellm.llms.custom_httpx.http_handler import (
@@ -36,7 +38,7 @@ from ..base_aws_llm import (
     pop_aws_auth_params,
     run_aws_signing,
 )
-from ..common_utils import BedrockError
+from ..common_utils import BedrockError, split_bedrock_region_path
 from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
 from .amazon_titan_g1_transformation import AmazonTitanG1Config
 from .amazon_titan_multimodal_transformation import (
@@ -64,12 +66,21 @@ def _sign_get_request(
     return request.prepare()
 
 
+_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
+
+
+def _without_region_path(model: str) -> str:
+    region, model_id = split_bedrock_region_path(model)
+    return model if region is None else model_id
+
+
 class BedrockEmbedding(BaseAWSLLM):
     @overload
     def _load_credentials(
         self,
         optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
         bearer_token: None = None,
+        region_from_model: str | None = None,
     ) -> tuple[Credentials, str]: ...
 
     @overload
@@ -77,15 +88,18 @@ class BedrockEmbedding(BaseAWSLLM):
         self,
         optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
         bearer_token: str,
+        region_from_model: str | None = None,
     ) -> tuple[None, str]: ...
 
     def _load_credentials(
         self,
         optional_params: dict,
         bearer_token: str | None = None,
+        region_from_model: str | None = None,
     ) -> tuple[Credentials | None, str]:
         auth_params: Final = pop_aws_auth_params(optional_params)
-        aws_region_name = optional_params.pop("aws_region_name", None)
+        explicit_region_name: Final = optional_params.pop("aws_region_name", None)
+        aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name
 
         ### SET REGION NAME ###
         if aws_region_name is None:
@@ -184,14 +198,17 @@ class BedrockEmbedding(BaseAWSLLM):
         provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
         is_async_invoke: bool | None = False,
         batch_data: list[dict] | None = None,
+        aws_region_name: str | None = None,
     ) -> EmbeddingResponse | None:
         """
         Transforms the response from the Bedrock embedding provider to the OpenAI format.
         """
         returned_response: EmbeddingResponse | None = None
+        bedrock_model: Final = _without_region_path(model)
+        is_async_job: Final = bool(is_async_invoke) and len(response_list) == 1 and "invocationArn" in response_list[0]
 
         # Handle async invoke responses (single response with invocationArn)
-        if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]:
+        if is_async_job:
             if provider == "twelvelabs":
                 returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response(
                     response=response_list[0], model=model
@@ -227,15 +244,15 @@ class BedrockEmbedding(BaseAWSLLM):
                 )
         else:
             # Handle regular invoke responses
-            if model == "amazon.titan-embed-image-v1":
+            if bedrock_model == "amazon.titan-embed-image-v1":
                 returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response(
                     response_list=response_list, model=model, batch_data=batch_data
                 )
-            elif model == "amazon.titan-embed-text-v1":
+            elif bedrock_model == "amazon.titan-embed-text-v1":
                 returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
-            elif model == "amazon.titan-embed-text-v2:0":
+            elif bedrock_model == "amazon.titan-embed-text-v2:0":
                 returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model)
-            elif model == "amazon.titan-embed-g1-text-02":
+            elif bedrock_model == "amazon.titan-embed-g1-text-02":
                 returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
             elif provider == "twelvelabs":
                 returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response(
@@ -251,6 +268,8 @@ class BedrockEmbedding(BaseAWSLLM):
         ##########################################################
         if returned_response is None:
             raise Exception(f"Unable to map model response to known provider format. model={model}")
+        if aws_region_name is not None and not is_async_job:
+            set_hidden_param(returned_response, "region_name", aws_region_name)
         return returned_response
 
     def _single_func_embeddings(
@@ -319,6 +338,7 @@ class BedrockEmbedding(BaseAWSLLM):
             provider=provider,
             is_async_invoke=is_async_invoke,
             batch_data=batch_data,
+            aws_region_name=aws_region_name,
         )
 
     async def _async_single_func_embeddings(
@@ -390,6 +410,7 @@ class BedrockEmbedding(BaseAWSLLM):
             provider=provider,
             is_async_invoke=is_async_invoke,
             batch_data=batch_data,
+            aws_region_name=aws_region_name,
         )
 
     def embeddings(
@@ -409,26 +430,29 @@ class BedrockEmbedding(BaseAWSLLM):
         litellm_params: dict,
         api_key: str | None = None,
     ) -> EmbeddingResponse:
+        model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None
+        has_async_invoke: Final = "async_invoke/" in model
+        model_without_async_route: Final = model.replace("async_invoke/", "", 1)
+        path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None
+        bedrock_model: Final = _without_region_path(model_without_async_route)
         credentials, aws_region_name = self._load_credentials(
-            optional_params, bearer_token=bedrock_bearer_token(api_key)
+            optional_params,
+            bearer_token=bedrock_bearer_token(api_key),
+            region_from_model=self._get_aws_region_from_model_arn(model_id_param) or path_region,
         )
 
         ### TRANSFORMATION ###
-        unencoded_model_id: Final = optional_params.pop("model_id", None) or model  # default to model if not passed
+        unencoded_model_id: Final = model_id_param or bedrock_model
         modelId: Final = urllib.parse.quote(unencoded_model_id, safe="")
         aws_region_name = self._get_aws_region_name(
             optional_params={"aws_region_name": aws_region_name},
-            model=model,
+            model=bedrock_model,
             model_id=unencoded_model_id,
         )
-        # Check async invoke needs to be used
-        has_async_invoke: Final = "async_invoke/" in model
-        if has_async_invoke:
-            model = model.replace("async_invoke/", "", 1)
-        provider: Final = self.get_bedrock_embedding_provider(model)
+        provider: Final = self.get_bedrock_embedding_provider(bedrock_model)
         if provider is None:
             raise Exception(
-                f"Unable to determine bedrock embedding provider for model: {model}. "
+                f"Unable to determine bedrock embedding provider for model: {bedrock_model}. "
                 f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
             )
         inference_params = copy.deepcopy(optional_params)
@@ -441,9 +465,9 @@ class BedrockEmbedding(BaseAWSLLM):
         batch_data: list | None = None
         if provider == "cohere":
             data = BedrockCohereEmbeddingConfig().transform_request(
-                model=model, input=input, inference_params=inference_params
+                model=bedrock_model, input=input, inference_params=inference_params
             )
-        elif provider == "amazon" and model in [
+        elif provider == "amazon" and bedrock_model in [
             "amazon.titan-embed-image-v1",
             "amazon.titan-embed-text-v1",
             "amazon.titan-embed-text-v2:0",
@@ -451,28 +475,28 @@ class BedrockEmbedding(BaseAWSLLM):
         ]:
             batch_data = []
             for i in input:
-                if model == "amazon.titan-embed-image-v1":
+                if bedrock_model == "amazon.titan-embed-image-v1":
                     transformed_request: AmazonEmbeddingRequest = (
                         AmazonTitanMultimodalEmbeddingG1Config().transform_request(
                             input=i, inference_params=inference_params
                         )
                     )
-                elif model == "amazon.titan-embed-text-v1":
+                elif bedrock_model == "amazon.titan-embed-text-v1":
                     transformed_request = AmazonTitanG1Config().transform_request(
                         input=i, inference_params=inference_params
                     )
-                elif model == "amazon.titan-embed-text-v2:0":
+                elif bedrock_model == "amazon.titan-embed-text-v2:0":
                     transformed_request = AmazonTitanV2Config().transform_request(
                         input=i, inference_params=inference_params
                     )
-                elif model == "amazon.titan-embed-g1-text-02":
+                elif bedrock_model == "amazon.titan-embed-g1-text-02":
                     transformed_request = AmazonTitanG1Config().transform_request(
                         input=i, inference_params=inference_params
                     )
                 else:
                     raise Exception(
                         "Unmapped model. Received={}. Expected={}".format(
-                            model,
+                            bedrock_model,
                             [
                                 "amazon.titan-embed-image-v1",
                                 "amazon.titan-embed-text-v1",
@@ -485,7 +509,7 @@ class BedrockEmbedding(BaseAWSLLM):
         elif provider == "twelvelabs":
             batch_data = []
             for i in input:
-                twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request(
+                twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request(
                     input=i,
                     inference_params=inference_params,
                     async_invoke_route=has_async_invoke,
@@ -527,7 +551,7 @@ class BedrockEmbedding(BaseAWSLLM):
                     extra_headers=extra_headers,
                     endpoint_url=endpoint_url,
                     aws_region_name=aws_region_name,
-                    model=model,
+                    model=model_without_async_route,
                     logging_obj=logging_obj,
                     api_key=api_key,
                     provider=provider,
@@ -541,7 +565,7 @@ class BedrockEmbedding(BaseAWSLLM):
                 extra_headers=extra_headers,
                 endpoint_url=endpoint_url,
                 aws_region_name=aws_region_name,
-                model=model,
+                model=model_without_async_route,
                 logging_obj=logging_obj,
                 api_key=api_key,
                 provider=provider,
@@ -570,8 +594,9 @@ class BedrockEmbedding(BaseAWSLLM):
         ## ROUTING ##
         # Convert CaseInsensitiveDict to regular dict for httpx compatibility
         headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {}
+        set_hidden_param(model_response, "region_name", aws_region_name)
         return cohere_embedding(
-            model=model,
+            model=model_without_async_route,
             input=input,
             model_response=model_response,
             logging_obj=logging_obj,

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py

@@ -7,6 +7,7 @@ from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request
 from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.constants import (
     BEDROCK_MIN_THINKING_BUDGET_TOKENS,
@@ -549,6 +550,9 @@ class AmazonAnthropicClaudeMessagesConfig(
             "edits": retained_edits,
         }
 
+    def strips_beta_gated_fields_from_final_headers(self) -> bool:
+        return False
+
     def _get_bedrock_invoke_anthropic_beta_headers(
         self,
         model: str,
@@ -584,9 +588,7 @@ class AmazonAnthropicClaudeMessagesConfig(
             ),
             custom_llm_provider="bedrock",
             is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
-            is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used(
-                anthropic_messages_request.get("thinking")
-            ),
+            thinking=anthropic_messages_request.get("thinking"),
             is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used(
                 outgoing_messages_typed
             ),
@@ -757,7 +759,9 @@ class AmazonAnthropicClaudeMessagesConfig(
                 ),
                 "display": "updates",
             }
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)
+        self._normalize_system_role_messages(
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
         #########################################################
         ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ###
         #########################################################
@@ -825,6 +829,20 @@ class AmazonAnthropicClaudeMessagesConfig(
         if filtered_betas:
             anthropic_messages_request["anthropic_beta"] = filtered_betas
 
+        final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request)  # pyright: ignore[reportUnknownArgumentType]  # built from the base signature's bare dict
+        gated: Final = strip_beta_gated_native_request(
+            {},
+            final_request,
+            final_request.get("messages"),
+            body_betas=filtered_betas,
+            drop_params=getattr(litellm_params, "drop_params", None),
+        )
+        log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke")
+        if gated.thinking_stripped:
+            anthropic_messages_request["thinking"] = gated.thinking
+        if gated.messages_stripped:
+            anthropic_messages_request["messages"] = list(gated.messages)
+
         outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request)
         if outbound_tools is not None:
             anthropic_messages_request["tools"] = outbound_tools

litellm/llms/cohere/embed/transformation.py

@@ -10,11 +10,13 @@ Convers
 Docs - https://docs.cohere.com/v2/reference/embed
 """
 
-from collections.abc import Iterable, Mapping, Sized
-from typing import Final, Protocol, cast
+from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized
+from enum import Enum
+from itertools import chain, starmap
+from typing import Final, Protocol, TypeAlias, cast
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 
 import litellm
 from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
@@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded
 from ..common_utils import CohereError
 
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
-_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True))
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
 class _SupportsEncode(Protocol):
     """Tokenizer handle: the embedding usage path only encodes text to measure its token length."""
 
-    def encode(self, text: str, /) -> Sized: ...
+    def encode_ordinary(self, text: str, /) -> Sized: ...
+
+
+class CohereImageTokenBilling(Enum):
+    SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens"
+    FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens"
+
+
+class CohereEmbeddingRowShape(Enum):
+    OPENAI = "openai"
+    TYPED = "typed"
+
+
+class _CohereBilledUnits(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    input_tokens: int | None = None
+    image_tokens: int | None = None
+    images: int | None = None
+
+
+class _CohereEmbedMeta(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits)
+
+
+_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object]
+
+
+class _CohereEmbedBody(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    embeddings: _CohereEmbeddings
+    meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta)
+
+
+def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]:
+    return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors))
+
+
+def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]:
+    return ({**row, "type": embedding_type} for row in _embedding_rows(vectors))
+
+
+def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]:
+    if not isinstance(embeddings, Mapping):
+        return _embedding_rows(embeddings)
+    match row_shape:
+        case CohereEmbeddingRowShape.OPENAI:
+            return chain.from_iterable(map(_embedding_rows, embeddings.values()))
+        case CohereEmbeddingRowShape.TYPED:
+            return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items()))
+
+
+def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage:
+    return Usage(
+        prompt_tokens=prompt_tokens,
+        completion_tokens=0,
+        total_tokens=prompt_tokens,
+        prompt_tokens_details=prompt_tokens_details,
+    )
+
+
+def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage:
+    match image_billing:
+        case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS:
+            image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0)
+            return _prompt_usage(
+                (billed_units.input_tokens or 0) + image_units,
+                PromptTokensDetailsWrapper(
+                    text_tokens=billed_units.input_tokens,
+                    image_tokens=image_units or None,
+                    image_count=billed_units.images,
+                ),
+            )
+        case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS:
+            return _prompt_usage(billed_units.input_tokens or 0, None)
+
+
+def _cohere_embed_usage(
+    billed_units: _CohereBilledUnits,
+    input_texts: Iterable[str],
+    encoding: _SupportsEncode,
+    image_billing: CohereImageTokenBilling,
+) -> Usage:
+    if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None:
+        return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None)
+    return _billed_usage(billed_units, image_billing)
+
+
+def cohere_embedding_response(
+    body: Mapping[str, object],
+    model: str | None,
+    encoding: _SupportsEncode,
+    input_texts: Iterable[str],
+    image_billing: CohereImageTokenBilling,
+    row_shape: CohereEmbeddingRowShape,
+) -> EmbeddingResponse:
+    cohere_body: Final = _CohereEmbedBody.model_validate(body)
+    return EmbeddingResponse(
+        model=model,
+        data=list(_response_rows(cohere_body.embeddings, row_shape)),
+        usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing),
+    )
 
 
 class CohereEmbeddingConfig(BaseEmbeddingConfig):
@@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
             ),
         )
 
-    def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage:
-        input_tokens = 0
-
-        text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens")
-
-        image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images")
-
-        prompt_tokens_details: PromptTokensDetailsWrapper | None = None
-        if image_tokens is None and text_tokens is None:
-            for text in input:
-                input_tokens += len(encoding.encode(text))
-        else:
-            prompt_tokens_details = PromptTokensDetailsWrapper(
-                image_tokens=image_tokens,
-                text_tokens=text_tokens,
-            )
-            if image_tokens:
-                input_tokens += image_tokens
-            if text_tokens:
-                input_tokens += text_tokens
-
-        return Usage(
-            prompt_tokens=input_tokens,
-            completion_tokens=0,
-            total_tokens=input_tokens,
-            prompt_tokens_details=prompt_tokens_details,
-        )
-
-    def _transform_response(
+    def transform_native_embed_response(
         self,
         response: httpx.Response,
         api_key: str | None,
         logging_obj: LiteLLMLoggingObj,
-        data: dict | CohereEmbeddingRequest,
-        model_response: EmbeddingResponse,
+        data: Mapping[str, object] | CohereEmbeddingRequest,
         model: str,
         encoding: _SupportsEncode,
-        input: list,
+        input: Sequence[str],
+        row_shape: CohereEmbeddingRowShape,
     ) -> EmbeddingResponse:
         response_json: Final = _DECODED_JSON.validate_python(response.json())
         ## LOGGING
@@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
             additional_args={"complete_input_dict": data},
             original_response=response_json,
         )
-        """
-            response 
-            {
-                'object': "list",
-                'data': [
-                
-                ]
-                'model', 
-                'usage'
-            }
-        """
-        payload: Final = _JSON_OBJECT.validate_python(response_json)
-        embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"])
-        output_data: Final = []
-        for k, embedding_list in embeddings.items():
-            for idx, embedding in enumerate(embedding_list):
-                output_data.append({"object": "embedding", "index": idx, "embedding": embedding})
-        model_response.object = "list"
-        model_response.data = output_data
-        model_response.model = model
-        input_tokens = 0
-        for text in input:
-            input_tokens += len(encoding.encode(text))
-
-        setattr(
-            model_response,
-            "usage",
-            self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))),
+        return cohere_embedding_response(
+            _JSON_OBJECT.validate_python(response_json),
+            model,
+            encoding,
+            input,
+            CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS,
+            row_shape,
         )
 
-        return model_response
-
     def transform_embedding_response(
         self,
         model: str,
@@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
         optional_params: dict,
         litellm_params: dict,
     ) -> EmbeddingResponse:
-        return self._transform_response(
+        return self.transform_native_embed_response(
             response=raw_response,
             api_key=api_key,
             logging_obj=logging_obj,
             data=request_data,
-            model_response=model_response,
             model=model,
             encoding=litellm.encoding,
             input=logging_obj.model_call_details["input"],
+            row_shape=CohereEmbeddingRowShape.OPENAI,
         )
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2581,6 +2581,7 @@ class BaseLLMHTTPHandler:
                 "preset_cache_key": None,
                 "stream_response": {},
                 "model_info": kwargs.get("model_info"),
+                **litellm_params.per_second_pricing(),
                 **vertex_location_params,
                 **anthropic_messages_optional_request_params,
             },

litellm/llms/databricks/chat/transformation.py

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
 import httpx
 from pydantic import BaseModel, ConfigDict, TypeAdapter
 
+from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     handle_invalid_parallel_tool_calls,
@@ -514,6 +515,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         self.update_optional_params_with_thinking_tokens(
             non_default_params=non_default_params, optional_params=mapped_params
         )
+        gated_thinking: Final = strip_beta_gated_thinking(
+            mapped_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # mapped_params is the base config's bare dict
+            frozenset(),
+            drop_params=drop_params,
+        )
+        if gated_thinking.stripped:
+            log_beta_gated_strips(gated_thinking.stripped, model=model, provider="databricks")
+            mapped_params["thinking"] = gated_thinking.thinking
 
         return mapped_params
 

litellm/llms/databricks/messages/transformation.py

@@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig)
             ("content-type" if key.lower() == "content-type" else key): value
             for key, value in {**auth_headers, **client_anthropic_headers}.items()
         }
-        finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the shared helper returns a bare dict  # mutable-ok: becomes the outbound headers
+        finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the shared helper returns a bare dict  # mutable-ok: becomes the outbound headers
             headers=normalized_headers,
             optional_params=optional_params,
             messages=messages,
@@ -191,6 +191,8 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig)
             litellm_params=litellm_params,
             headers=headers,
         )
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
+        self._normalize_system_role_messages(  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
         self._remove_scope_from_cache_control(anthropic_messages_request)  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
         return anthropic_messages_request

litellm/llms/openai/common_utils.py

@@ -105,6 +105,19 @@ def is_openai_backed_api_base(api_base: str) -> bool:
     return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
 
 
+_OPENAI_AGENTS_API_RESOURCES: Final = frozenset({"agents", "vaults"})
+
+
+def openai_api_resource(path: str) -> str | None:
+    segments: Final = tuple(segment for segment in path.split("/") if segment)
+    resource_segments: Final = segments[1:] if segments[:1] == ("v1",) else segments
+    return next(iter(resource_segments), None)
+
+
+def is_openai_agents_api_path(path: str) -> bool:
+    return openai_api_resource(path) in _OPENAI_AGENTS_API_RESOURCES
+
+
 class OpenAIError(BaseLLMException):
     def __init__(
         self,

litellm/llms/openai_like/providers.json

@@ -270,6 +270,13 @@
       "/v1/responses"
     ]
   },
+  "kyma": {
+    "base_url": "https://kymaapi.com/v1",
+    "api_key_env": "KYMA_API_KEY",
+    "provider_info": {
+      "supports_function_calling": true
+    }
+  },
   "streamlake": {
     "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
     "api_key_env": "STREAMLAKE_API_KEY",

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py

@@ -211,6 +211,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
         if messages_carry_output_config(messages):
             beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
 
+        beta_values.update(
+            anthropic_model_info.thinking_betas(
+                optional_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # optional_params is the base config's bare dict
+            )
+        )
+
         if beta_values:
             request_headers["anthropic-beta"] = ",".join(beta_values)
 
@@ -245,7 +251,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
             headers=headers,
         )
 
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)
+        self._normalize_system_role_messages(
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
 
         self._remove_scope_from_cache_control(anthropic_messages_request)
 

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py

@@ -145,6 +145,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
             file_id_used=self.is_file_id_used(messages),
             mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
             custom_llm_provider="vertex_ai",
+            thinking=optional_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # optional_params is the base config's bare dict
         )
 
         beta_set: Final = set(auto_betas)

litellm/provider_endpoints_support_backup.json

@@ -5327,6 +5327,32 @@
         "generateContent": false
       }
     },
+    "kyma": {
+      "display_name": "Kyma API (`kyma`)",
+      "url": "https://docs.litellm.ai/docs/providers/kyma",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": false,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
+    },
     "bigmodel": {
       "display_name": "BigModel (Zhipu AI) (`bigmodel`)",
       "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction",

litellm/proxy/_types.py

@@ -3748,6 +3748,7 @@ class SpendLogsMetadata(TypedDict):
     internal_call_origin: InternalCallOrigin | None
     billed_retry_attempt: ReadOnly[NotRequired[bool | None]]
     litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
+    is_silent_experiment: ReadOnly[NotRequired[bool | None]]
     guardrail_information: list[StandardLoggingGuardrailInformation] | None
     eval_information: Any | None
     status: StandardLoggingPayloadStatus

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -2,7 +2,7 @@
 Unified /v1/messages endpoint - (Anthropic Spec)
 """
 
-from collections.abc import Mapping
+from collections.abc import AsyncGenerator, Mapping
 from types import MappingProxyType
 from typing import Final
 
@@ -19,12 +19,10 @@ from litellm.anthropic_interface.exceptions import (
 )
 from litellm.integrations.custom_guardrail import ModifyResponseException
 from litellm.litellm_core_utils.tool_fields import first_validation_error
+from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
 from litellm.llms.anthropic.messages.context_management import (
     AnthropicContextManagementError,
 )
-from litellm.llms.base_llm.guardrail_translation.utils import (
-    blocked_response_usage as _blocked_response_usage,
-)
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_request_processing import (
@@ -44,6 +42,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (  # noqa: F401  # leg
 from litellm.proxy.common_utils.openai_error_payload import (
     LITELLM_CALL_ID_HEADER,
     error_status_code,
+    litellm_call_id_headers,
     openai_error_param,
     openai_error_type,
     with_litellm_call_id,
@@ -58,6 +57,8 @@ from litellm.router import Router
 
 router: Final = APIRouter()
 
+_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
+
 
 def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail:
     if not exc.provider_specific_fields:
@@ -229,44 +230,22 @@ async def anthropic_response(
             request_data=_data,
         )
 
-        # Create Anthropic-formatted response with violation message
-        import uuid
+        block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS
+        block_handler: Final = AnthropicMessagesHandler()
 
-        from litellm.types.utils import AnthropicMessagesResponse
+        if data.get("stream") is True:
+            block_chunks: Final = block_handler.build_block_sse_chunks(e)
 
-        # Report the blocked LLM response's real token usage (carried on the
-        # exception) instead of discarding it; zero for pre-call blocks.
-        _usage: Final = _blocked_response_usage(e.original_response)
-
-        _anthropic_response: Final = AnthropicMessagesResponse(
-            id=f"msg_{uuid.uuid4()}",
-            type="message",
-            role="assistant",
-            content=[{"type": "text", "text": e.message}],
-            model=e.model,
-            stop_reason="end_turn",
-            usage=_usage,
-        )
-
-        if data.get("stream", None) is not None and data["stream"] is True:
-            # For streaming, use the standard SSE data generator
-            async def _passthrough_stream_generator():
-                yield _anthropic_response
-
-            selected_data_generator: Final = ProxyBaseLLMRequestProcessing.async_sse_data_generator(
-                response=_passthrough_stream_generator(),
-                user_api_key_dict=user_api_key_dict,
-                request_data=_data,
-                proxy_logging_obj=proxy_logging_obj,
-            )
+            async def _blocked_stream() -> AsyncGenerator[str, None]:
+                for chunk in block_chunks:
+                    yield chunk.decode()
 
             return await create_response(
-                generator=selected_data_generator,
-                media_type="text/event-stream",
-                headers={},
+                generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers
             )
 
-        return _anthropic_response
+        fastapi_response.headers.update(block_headers)
+        return block_handler.build_block_response(e)
     except AnthropicContextManagementError as e:
         if e.status_code >= 500:
             # Server-side polyfill failures hit the failure hook for spend/alert

litellm/proxy/health_check_utils/shared_health_check_manager.py

@@ -2,7 +2,10 @@ import asyncio
 import json
 import time
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final
+
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._internal_context import with_service_target
 from litellm._logging import verbose_proxy_logger
@@ -19,6 +22,17 @@ if TYPE_CHECKING:
     from litellm.router import Router
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CachedHealthCheckResults(TypedDict):
+    healthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]]
+    unhealthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]]
+    timestamp: NotRequired[ReadOnly[int | float]]
+    checked_by: NotRequired[ReadOnly[object]]
+
+
+_CACHED_HEALTH_CHECK_RESULTS: Final = TypeAdapter(_CachedHealthCheckResults)
+
+
 class SharedHealthCheckManager:
     """
     Manager for coordinating health checks across multiple pods using Redis.
@@ -109,7 +123,7 @@ class SharedHealthCheckManager:
             verbose_proxy_logger.error("Error releasing health check lock: %s", str(e))
 
     @with_service_target(HEALTH_CHECKS_TARGET)
-    async def get_cached_health_check_results(self) -> dict[str, Any] | None:
+    async def get_cached_health_check_results(self) -> _CachedHealthCheckResults | None:
         """
         Get cached health check results from Redis.
 
@@ -127,10 +141,9 @@ class SharedHealthCheckManager:
                 return None
 
             # Parse the cached data
-            if isinstance(cached_data, str):
-                cached_results = json.loads(cached_data)
-            else:
-                cached_results = cached_data
+            cached_results: Final = _CACHED_HEALTH_CHECK_RESULTS.validate_python(
+                json.loads(cached_data) if isinstance(cached_data, str) else cached_data
+            )
 
             # Check if the cache is still valid
             cache_timestamp: Final = cached_results.get("timestamp", 0)

litellm/proxy/management_endpoints/fallback_management_endpoints.py

@@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.model_checks import get_all_fallbacks
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.utils import PrismaClient, evict_config_param, invalidate_config_param
+from litellm.proxy.utils import PrismaClient, invalidate_config_param
 
 if TYPE_CHECKING:
     from fastapi import APIRouter, Depends, HTTPException, status
@@ -48,22 +48,50 @@ FallbackRule = dict[str, list[str]]
 StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")]
 _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
 _JSON_RULES: Final = TypeAdapter(list[JsonValue])
+_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
 
 
 def _rule_covers(entry: StoredFallback, model: str) -> bool:
     return isinstance(entry, dict) and model in entry
 
 
-async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]:
-    await evict_config_param(ROUTER_SETTINGS_PARAM)
-    config: Final = await proxy_config.get_config()
-    return config.get(ROUTER_SETTINGS_PARAM, {})
+async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None:
+    await proxy_config.get_config()
 
 
-def _reject_config_owned_rules(proxy_config: "ProxyConfig", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
+async def _stored_router_settings(prisma_client: PrismaClient) -> Mapping[str, JsonValue]:
+    try:
+        row: Final = await ConfigRepository(prisma_client, use_writer=True).get_param(ROUTER_SETTINGS_PARAM)
+    except json.JSONDecodeError:
+        return {}
+    if row is None or not isinstance(row.param_value, Mapping):
+        return {}
+    return _STORED_ROUTER_SETTINGS.validate_python(row.param_value)
+
+
+def _editable_rules(
+    proxy_config: "ProxyConfig", stored_settings: Mapping[str, JsonValue], fallback_key: str
+) -> list[StoredFallback]:
+    file_settings: Final = proxy_config.router_settings
+    if not file_settings.owned_by_config(fallback_key):
+        return _STORED_FALLBACKS.validate_python(stored_settings.get(fallback_key) or [])
+    file_rules: Final = file_settings.get(fallback_key)
+    return _STORED_FALLBACKS.validate_python(file_rules if isinstance(file_rules, list) else [])
+
+
+async def _write_rules(
+    prisma_client: PrismaClient,
+    proxy_config: "ProxyConfig",
+    stored_settings: Mapping[str, JsonValue],
+    fallback_key: str,
+    rules: Sequence[StoredFallback],
+) -> None:
     proxy_config.reject_config_owned_writes(
         section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)}
     )
+    if proxy_config.router_settings.owned_by_config(fallback_key):
+        return
+    await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules})
 
 
 async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None:
@@ -162,7 +190,8 @@ async def create_fallback(
                 },
             )
 
-        router_settings: Final = await _router_settings_fresh_from_db(proxy_config)
+        await _reload_config_file_ownership(proxy_config)
+        router_settings: Final = await _stored_router_settings(prisma_client)
 
         # Get the appropriate fallback list based on type
         fallback_key = "fallbacks"
@@ -172,7 +201,7 @@ async def create_fallback(
             fallback_key = "content_policy_fallbacks"
 
         # Get existing fallbacks
-        existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or [])
+        existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key)
 
         # Update or add the fallback configuration
         fallback_updated = False
@@ -187,12 +216,7 @@ async def create_fallback(
             # Add new fallback
             existing_fallbacks.append({data.model: data.fallback_models})
 
-        _reject_config_owned_rules(proxy_config, fallback_key, existing_fallbacks)
-
-        # Update router settings
-        router_settings[fallback_key] = existing_fallbacks
-
-        await _persist_router_settings(prisma_client, router_settings)
+        await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks)
 
         # Update the in-memory router configuration
         setattr(llm_router, fallback_key, existing_fallbacks)
@@ -320,7 +344,8 @@ async def delete_fallback(
                 },
             )
 
-        router_settings: Final = await _router_settings_fresh_from_db(proxy_config)
+        await _reload_config_file_ownership(proxy_config)
+        router_settings: Final = await _stored_router_settings(prisma_client)
 
         # Get the appropriate fallback list based on type
         fallback_key = "fallbacks"
@@ -330,7 +355,7 @@ async def delete_fallback(
             fallback_key = "content_policy_fallbacks"
 
         # Get existing fallbacks
-        existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or [])
+        existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key)
 
         updated_fallbacks: Final = [rule for rule in existing_fallbacks if not _rule_covers(rule, model)]
 
@@ -340,12 +365,7 @@ async def delete_fallback(
                 detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"},
             )
 
-        _reject_config_owned_rules(proxy_config, fallback_key, updated_fallbacks)
-
-        # Update router settings
-        router_settings[fallback_key] = updated_fallbacks
-
-        await _persist_router_settings(prisma_client, router_settings)
+        await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks)
 
         # Update the in-memory router configuration
         setattr(llm_router, fallback_key, updated_fallbacks)

litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py

@@ -43,6 +43,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.proxy.db.db_span import db_span
 from litellm.proxy.mcp_server.oauth_utils import canonicalize_url_identity
 from litellm.proxy.utils import PrismaClient
+from litellm.repositories.table_repositories import MCPServerRepository
 
 # The actor the removed discovery write-back stamped rows with.
 _DISCOVERY_ACTOR: Final = "mcp_oauth_discovery"
@@ -106,7 +107,7 @@ def _is_stamped_issuer_row(row: _MCPServerRow) -> bool:
 
 async def backfill_discovery_stamped_issuers(prisma_client: PrismaClient) -> int:
     """Clear gateway-written issuer stamps, returning the number of rows healed."""
-    candidate_rows: Final[list[_MCPServerRow]] = await prisma_client.db.litellm_mcpservertable.find_many(
+    candidate_rows: Final = await MCPServerRepository(prisma_client).table.find_many(
         where={
             "updated_by": _DISCOVERY_ACTOR,
             "auth_type": {"in": list(_AUTH_TYPES_WITH_ISSUER_ANCHORING)},

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -68,6 +68,7 @@ from litellm.llms.deepgram.common_utils import (
 from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
 from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
 from litellm.llms.openai.common_utils import OpenAIError as LiteLLMOpenAIError
+from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_api_resource
 from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base
 from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@@ -3530,6 +3531,10 @@ async def deepgram_listen_websocket_route(
     )
 
 
+_OPENAI_BETA_HEADER: Final = "openai-beta"
+_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"})
+
+
 class BaseOpenAIPassThroughHandler:
     @staticmethod
     async def base_openai_pass_through_handler(
@@ -3540,7 +3545,7 @@ class BaseOpenAIPassThroughHandler:
         base_target_url: str,
         api_key: str | None,
         custom_llm_provider: litellm.LlmProviders,
-        extra_headers: dict | None = None,
+        extra_headers: Mapping[str, str] | None = None,
     ):
         encoded_endpoint = httpx.URL(endpoint).path
         # Ensure endpoint starts with '/' for proper URL construction
@@ -3565,7 +3570,11 @@ class BaseOpenAIPassThroughHandler:
             endpoint=endpoint,
             target=str(updated_url),
             custom_headers=BaseOpenAIPassThroughHandler._assemble_headers(
-                api_key=api_key, request=request, extra_headers=extra_headers
+                api_key=api_key,
+                endpoint=endpoint,
+                request=request,
+                custom_llm_provider=custom_llm_provider,
+                extra_headers=extra_headers,
             ),
             is_streaming_request=is_streaming_request,
             custom_llm_provider=(
@@ -3587,27 +3596,41 @@ class BaseOpenAIPassThroughHandler:
     _base_openai_pass_through_handler = base_openai_pass_through_handler
 
     @staticmethod
-    def _append_openai_beta_header(headers: dict, request: Request) -> dict:
-        """
-        Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request
-        """
-        if RouteChecks.is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers:
-            headers["OpenAI-Beta"] = "assistants=v2"
-        return headers
+    def _openai_beta(endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders) -> str | None:
+        if (
+            custom_llm_provider == litellm.LlmProviders.OPENAI
+            and openai_api_resource(endpoint) in _CLIENT_OPENAI_BETA_RESOURCES
+        ):
+            return request.headers.get(_OPENAI_BETA_HEADER) or (
+                "agents=v1" if is_openai_agents_api_path(endpoint) else None
+            )
+        return "assistants=v2" if RouteChecks.is_assistants_api_request(request) else None
 
     @staticmethod
-    def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict:
-        base_headers = {}
-        if api_key is not None:
-            base_headers = {
-                "authorization": f"Bearer {api_key}",
-                "api-key": f"{api_key}",
-            }
-        if extra_headers is not None:
-            base_headers.update(extra_headers)
+    def _append_openai_beta_header(
+        headers: Mapping[str, str], endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders
+    ) -> dict[str, str]:
+        if any(name.lower() == _OPENAI_BETA_HEADER for name in headers):
+            return {**headers}
+        openai_beta: Final = BaseOpenAIPassThroughHandler._openai_beta(endpoint, request, custom_llm_provider)
+        return {**headers} if openai_beta is None else {**headers, _OPENAI_BETA_HEADER: openai_beta}
+
+    @staticmethod
+    def _assemble_headers(
+        api_key: str | None,
+        endpoint: str,
+        request: Request,
+        custom_llm_provider: litellm.LlmProviders,
+        extra_headers: Mapping[str, str] | None = None,
+    ) -> dict[str, str]:
+        api_key_headers: Final[Mapping[str, str]] = (
+            {} if api_key is None else {"authorization": f"Bearer {api_key}", "api-key": api_key}
+        )
         return BaseOpenAIPassThroughHandler._append_openai_beta_header(
-            headers=base_headers,
+            headers={**api_key_headers, **(extra_headers or {})},
+            endpoint=endpoint,
             request=request,
+            custom_llm_provider=custom_llm_provider,
         )
 
 

litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py

@@ -1,10 +1,12 @@
 from datetime import datetime
 from typing import Final
+from urllib.parse import urlparse
 
 import httpx
 
 import litellm
 from litellm import stream_chunk_builder
+from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.hidden_params import set_hidden_param
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import (
@@ -16,7 +18,7 @@ from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig
 from litellm.llms.cohere.common_utils import (
     ModelResponseIterator as CohereModelResponseIterator,
 )
-from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig
+from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig, CohereEmbeddingRowShape
 from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     PassthroughStandardLoggingPayload,
@@ -29,6 +31,12 @@ from litellm.types.utils import (
 
 from .base_passthrough_logging_handler import BasePassthroughLoggingHandler
 
+COHERE_EMBED_PATHS: Final = ("/v1/embed", "/v2/embed")
+
+
+def is_cohere_embed_route(url_route: str) -> bool:
+    return urlparse(url_route).path.rstrip("/").endswith(COHERE_EMBED_PATHS)
+
 
 class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler):
     @property
@@ -82,28 +90,24 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler):
         """
         Handle Cohere passthrough logging with route detection and cost tracking.
         """
-        # Check if this is an embed endpoint
-        if "/v1/embed" in url_route and "/v1/embeddings" not in url_route:
+        if is_cohere_embed_route(url_route):
             model: Final = request_body.get("model", response_body.get("model", ""))
             try:
-                cohere_embed_config: Final = CohereEmbeddingConfig()
-                litellm_model_response = litellm.EmbeddingResponse()
                 handler_instance: Final = CoherePassthroughLoggingHandler()
 
                 input_texts = request_body.get("texts", [])
                 if not input_texts:
                     input_texts = request_body.get("input", [])
 
-                # Transform the response
-                litellm_model_response = cohere_embed_config.transform_response(
+                litellm_model_response: Final = CohereEmbeddingConfig().transform_native_embed_response(
                     response=httpx_response,
                     api_key="",
                     logging_obj=logging_obj,
                     data=request_body,
-                    model_response=litellm_model_response,
                     model=model,
                     encoding=litellm.encoding,
                     input=input_texts,
+                    row_shape=CohereEmbeddingRowShape.TYPED,
                 )
 
                 # Calculate cost using LiteLLM's cost calculator
@@ -153,20 +157,11 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     "result": litellm_model_response,
                     "kwargs": kwargs,
                 }
-            except Exception:
-                # For other routes (e.g., /v2/chat), fall back to chat handler
-                return super().passthrough_chat_handler(
-                    httpx_response=httpx_response,
-                    response_body=response_body,
-                    logging_obj=logging_obj,
-                    url_route=url_route,
-                    result=result,
-                    start_time=start_time,
-                    end_time=end_time,
-                    cache_hit=cache_hit,
-                    request_body=request_body,
-                    **kwargs,
+            except Exception as e:
+                verbose_proxy_logger.warning(
+                    "Could not price the Cohere passthrough embed response, logging it without a cost: %s", e
                 )
+                return {"result": None, "kwargs": kwargs}
 
         # For non-embed routes (e.g., /v2/chat), fall back to chat handler
         return super().passthrough_chat_handler(

litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py

@@ -10,7 +10,7 @@ from typing import Final, Literal, TypeAlias
 from urllib.parse import urlparse
 
 import httpx
-from pydantic import TypeAdapter
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -21,6 +21,8 @@ from litellm.litellm_core_utils.litellm_logging import (
     get_standard_logging_object_payload,
 )
 from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
+from litellm.llms.openai.common_utils import is_openai_agents_api_path
 from litellm.llms.openai.openai import OpenAIConfig
 from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@@ -38,7 +40,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     EndpointType,
     PassthroughStandardLoggingPayload,
 )
-from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, PassthroughCallTypes
+from litellm.types.utils import (
+    EmbeddingResponse,
+    ImageResponse,
+    LlmProviders,
+    PassthroughCallTypes,
+    StandardPassThroughResponseObject,
+)
 from litellm.utils import ModelResponse, TextCompletionResponse, convert_to_model_response_object
 
 # Hostnames that route to OpenAI-compatible APIs.
@@ -121,6 +129,52 @@ def _responses_sub_route(path: str) -> tuple[str, ...] | None:
     return None if anchor is None else tuple(segments[anchor + 1 :])
 
 
+_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
+
+
+class _AgentConfig(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model: str
+
+
+class _AgentsApiBody(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    agent: _AgentConfig
+
+
+class _AgentsApiSessionEvent(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    session: _AgentsApiBody
+
+
+_AGENTS_API_BODY: Final = TypeAdapter(_AgentsApiBody | _AgentConfig)
+
+
+def _agent_model(body: Mapping[str, object]) -> str | None:
+    try:
+        parsed: Final = _AGENTS_API_BODY.validate_python(body)
+    except ValidationError:
+        return None
+    return parsed.agent.model if isinstance(parsed, _AgentsApiBody) else parsed.model
+
+
+def _session_event_model(chunk: str) -> str | None:
+    event: Final = BaseModelResponseIterator.string_to_dict_parser(str_line=chunk)
+    if event is None:
+        return None
+    try:
+        return _AgentsApiSessionEvent.model_validate(event).session.agent.model
+    except ValidationError:
+        return None
+
+
+def _streamed_agent_model(all_chunks: Sequence[str]) -> str | None:
+    return next((model for chunk in all_chunks if (model := _session_event_model(chunk)) is not None), None)
+
+
 def _is_remote_high_detail_image(part: object) -> bool:
     if not isinstance(part, Mapping) or part.get("type") != "image_url":
         return False
@@ -220,14 +274,44 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                 return "billed"
 
     @staticmethod
-    def _label_unbilled_responses_call(
+    def is_openai_agents_route(url_route: str) -> bool:
+        return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path)
+
+    @staticmethod
+    def _unbilled_call_model(
+        url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object]
+    ) -> str | None:
+        if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route):
+            return _agent_model(response_body) or _agent_model(request_body)
+        response_model: Final = response_body.get("model")
+        if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(response_model, str):
+            return response_model
+        return None
+
+    @staticmethod
+    def _label_model(logging_obj: LiteLLMLoggingObj, model: str | None) -> None:
+        if model is not None:
+            logging_obj.model_call_details["model"] = model  # rebind-ok: spend rows read the model from here
+
+    @staticmethod
+    def _label_unbilled_call(
         logging_obj: LiteLLMLoggingObj,
         url_route: str,
+        request_body: Mapping[str, object],
         response_body: Mapping[str, object],
     ) -> None:
-        model: Final = response_body.get("model")
-        if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str):
-            logging_obj.model_call_details["model"] = model  # rebind-ok: spend rows read the model from here
+        OpenAIPassthroughLoggingHandler._label_model(
+            logging_obj, OpenAIPassthroughLoggingHandler._unbilled_call_model(url_route, request_body, response_body)
+        )
+
+    @staticmethod
+    def _log_agents_stream(
+        litellm_logging_obj: LiteLLMLoggingObj, request_body: Mapping[str, object], all_chunks: Sequence[str]
+    ) -> PassThroughEndpointLoggingTypedDict:
+        OpenAIPassthroughLoggingHandler._label_model(
+            litellm_logging_obj, _streamed_agent_model(all_chunks) or _agent_model(request_body)
+        )
+        return {"result": StandardPassThroughResponseObject(response="\n".join(all_chunks)), "kwargs": {}}
 
     @staticmethod
     def is_openai_embeddings_route(url_route: str) -> bool:
@@ -397,7 +481,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
         is_responses: Final = responses_billing != "unbilled"
 
         if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses):
-            OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body)
+            OpenAIPassthroughLoggingHandler._label_unbilled_call(
+                logging_obj,
+                url_route,
+                request_body=_STR_MAPPING.validate_python(request_body),
+                response_body=_STR_MAPPING.validate_python(response_body),
+            )
             return {
                 "result": None,
                 "kwargs": kwargs,
@@ -676,6 +765,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
         """
         Handle logging for collected OpenAI streaming chunks with cost tracking.
         """
+        if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route):
+            return OpenAIPassthroughLoggingHandler._log_agents_stream(
+                litellm_logging_obj, _STR_MAPPING.validate_python(request_body), all_chunks
+            )
         try:
             is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
             terminal_response: Final = (

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -22,6 +22,7 @@ from .llm_provider_handlers.assembly_passthrough_logging_handler import (
 )
 from .llm_provider_handlers.cohere_passthrough_logging_handler import (
     CoherePassthroughLoggingHandler,
+    is_cohere_embed_route,
 )
 from .llm_provider_handlers.cursor_passthrough_logging_handler import (
     CursorPassthroughLoggingHandler,
@@ -91,7 +92,7 @@ class PassThroughEndpointLogging:
         self.TRACKED_ANTHROPIC_ROUTES = ["/messages", "/v1/messages/batches"]
 
         # Cohere
-        self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"]
+        self.TRACKED_COHERE_ROUTES = ["/v2/chat"]
         self.assemblyai_passthrough_logging_handler = AssemblyAIPassthroughLoggingHandler()
 
         # Langfuse
@@ -536,13 +537,7 @@ class PassThroughEndpointLogging:
         return False
 
     def is_cohere_route(self, url_route: str) -> bool:
-        for route in self.TRACKED_COHERE_ROUTES:
-            if route not in url_route:
-                continue
-            if route == "/v1/embed" and "/v1/embeddings" in url_route:
-                continue
-            return True
-        return False
+        return is_cohere_embed_route(url_route) or any(route in url_route for route in self.TRACKED_COHERE_ROUTES)
 
     def is_assemblyai_route(self, url_route: str):
         parsed_url: Final = urlparse(url_route)
@@ -645,6 +640,7 @@ class PassThroughEndpointLogging:
             or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
             or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
             or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
         )
 
     def _set_cost_per_request(

litellm/proxy/plugin_routes.py

@@ -31,6 +31,8 @@ from typing import Final
 
 from cryptography.fernet import Fernet, InvalidToken
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._types import PluginConfig, SpecialHeaders, UserAPIKeyAuth
@@ -39,6 +41,18 @@ from litellm.types.llms.custom_http import httpxSpecialProvider
 
 router: Final = APIRouter()
 
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class PluginSessionClaim(TypedDict):
+    plugin: NotRequired[ReadOnly[object]]
+    user_id: NotRequired[ReadOnly[str]]
+    user_role: NotRequired[ReadOnly[str]]
+    exp: NotRequired[ReadOnly[int | float | str]]
+
+
+_PLUGIN_SESSION_CLAIM: Final = TypeAdapter(PluginSessionClaim)
+
+
 # Hop-by-hop headers (RFC 7230) and the litellm session cookie — never forwarded
 # to a plugin backend.  Credential headers are added on top per-request from the
 # canonical SpecialHeaders set so the plugin only ever authenticates via its own
@@ -155,7 +169,7 @@ def issue_plugin_session_claim(plugin_name: str, user_id: str | None, user_role:
     return _plugin_fernet(plugin_name).encrypt(json.dumps(claim).encode()).decode()
 
 
-def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict:
+def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> PluginSessionClaim:
     """Verify and decode a plugin session claim.
 
     Raises ValueError if the HMAC is invalid, the audience is wrong, or
@@ -163,7 +177,7 @@ def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict:
     """
     try:
         raw: Final = _plugin_fernet(plugin_name).decrypt(ciphertext.encode(), ttl=_CLAIM_TTL_SECONDS)
-        claim: Final = json.loads(raw)
+        claim: Final = _PLUGIN_SESSION_CLAIM.validate_python(json.loads(raw))
     except (InvalidToken, Exception) as exc:
         raise ValueError("Invalid, tampered, or expired plugin session claim") from exc
 

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -2023,6 +2023,34 @@
     ],
     "default_model_placeholder": "jina_ai/"
   },
+  {
+    "provider": "KYMA",
+    "provider_display_name": "Kyma API",
+    "litellm_provider": "kyma",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://kymaapi.com/v1",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "kyma/qwen3.7-flash"
+  },
   {
     "provider": "Laya",
     "provider_display_name": "Laya",

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     SETTLES_BACKGROUND_RESPONSE_KWARG,
     BackgroundResponseSettlement,
 )
-from litellm.types.router import DeploymentTypedDict, LiteLLM_Params
+from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag
 from litellm.types.utils import (
     BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
     PROMPT_CARRYING_GUARDRAIL_FIELDS,
@@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
         "used_client_oauth_token",
         KUBERNETES_POD_ROUTING_KEY,
         "litellm_roi_estimator",
+        "is_silent_experiment",
         BILLED_RETRY_ATTEMPT_METADATA_KEY,
         "user_api_key_user_email",
     )
@@ -227,6 +228,7 @@ def _get_spend_logs_metadata(
     azure_spillover: AzureSpillover | None = None,
     used_client_oauth_token: bool | None = None,
     kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None,
+    is_silent_experiment: bool = False,
 ) -> SpendLogsMetadata:
     if metadata is None:
         return SpendLogsMetadata(
@@ -294,6 +296,7 @@ def _get_spend_logs_metadata(
         used_client_oauth_token=used_client_oauth_token,
         kubernetes_pod_routing=kubernetes_pod_routing,
         litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
+        **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
         billed_retry_attempt=billed_retry_attempt,
         user_api_key_user_email=_stored_user_email(metadata),
     )
@@ -738,6 +741,11 @@ def get_logging_payload(
     # clean up litellm metadata
     clean_metadata = _get_spend_logs_metadata(
         persisted_metadata,
+        is_silent_experiment=(
+            standard_logging_payload is not None
+            and "is_silent_experiment" in standard_logging_payload["metadata"]
+            and standard_logging_payload["metadata"]["is_silent_experiment"]
+        ),
         applied_guardrails=(
             standard_logging_payload["metadata"].get("applied_guardrails", None)
             if standard_logging_payload is not None

litellm/responses/main.py

@@ -93,7 +93,6 @@ from litellm.types.utils import (
     AgenticLoopParams,
     CallTypes,
     CostPerToken,
-    CustomPricingLiteLLMParams,
     all_litellm_params,
 )
 from litellm.utils import (
@@ -566,12 +565,6 @@ async def aresponses_api_with_mcp(
     return response
 
 
-def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
-    return MappingProxyType(
-        litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
-    )
-
-
 class _DeploymentModelInfo(BaseModel):
     model_config = ConfigDict(frozen=True)
     id: object = None
@@ -1800,7 +1793,7 @@ def responses(
             optional_params=dict(responses_api_request_params),
             litellm_params={
                 **responses_api_request_params,
-                **_per_call_custom_pricing(litellm_params),
+                **litellm_params.custom_pricing_overrides(),
                 "aresponses": _is_async,
                 "litellm_call_id": litellm_call_id,
                 "model_info": kwargs.get("model_info"),

litellm/router.py

@@ -75,7 +75,7 @@ from typing_extensions import assert_never, overload
 import litellm
 import litellm.litellm_core_utils.exception_mapping_utils
 from litellm import get_secret_str
-from litellm._internal_context import service_target, with_service_target
+from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target
 from litellm._logging import verbose_router_logger
 from litellm._uuid import uuid
 from litellm.caching.caching import (
@@ -754,7 +754,7 @@ def _every_deployment_refused(
 
 
 def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
-    if silent_model is None:
+    if silent_model is None or in_mirrored_call():
         return ()
     try:
         targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model)
@@ -3372,16 +3372,17 @@ class Router:
         remaining_kwargs: Final = MappingProxyType(
             {key: value for key, value in silent_kwargs.items() if key != "stream"}
         )
-        response: Final = await self.acompletion(
-            model=silent_model,
-            messages=cast(list[AllMessageValues], messages),
-            stream=bool(silent_kwargs.get("stream", False)),
-            **remaining_kwargs,
-        )
-        if not isinstance(response, CustomStreamWrapper):
-            return
-        async for _ in response:
-            pass
+        with mirrored_call():
+            response: Final = await self.acompletion(
+                model=silent_model,
+                messages=cast(list[AllMessageValues], messages),
+                stream=bool(silent_kwargs.get("stream", False)),
+                **remaining_kwargs,
+            )
+            if not isinstance(response, CustomStreamWrapper):
+                return
+            async for _ in response:
+                pass
 
     def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
         """

litellm/types/integrations/custom_logger.py

@@ -97,9 +97,9 @@ class AgenticLoopRequestPatch(LiteLLMBaseModel):
 
     model: str | None = None
     messages: list[dict[str, Any]] | None = None
-    tools: list[dict[str, Any]] | None = None
+    tools: list[dict[str, object]] | None = None
     max_tokens: int | None = None
-    optional_params: dict[str, Any] = Field(default_factory=dict)
+    optional_params: dict[str, object] = Field(default_factory=dict)
     kwargs: dict[str, Any] = Field(default_factory=dict)
 
 

litellm/types/integrations/datadog_llm_obs.py

@@ -5,7 +5,7 @@ API Reference: https://docs.datadoghq.com/llm_observability/setup/api/?tab=examp
 """
 
 from collections.abc import Sequence
-from typing import Any, Literal
+from typing import Literal
 
 from typing_extensions import ReadOnly, TypedDict
 
@@ -16,7 +16,7 @@ class ToolCall(TypedDict, total=False):
     """A tool call on a message, as LLM Obs names its fields."""
 
     name: ReadOnly[str]
-    arguments: ReadOnly[dict[str, Any] | str]  # parsed object, or the raw string when it will not parse to one
+    arguments: ReadOnly[dict[str, object] | str]  # parsed object, or the raw string when it will not parse to one
     tool_id: ReadOnly[str]
     type: ReadOnly[str]
 
@@ -35,7 +35,7 @@ class ToolDefinition(TypedDict, total=False):
 
     name: ReadOnly[str]
     description: ReadOnly[str]
-    schema: ReadOnly[dict[str, Any]]
+    schema: ReadOnly[dict[str, object]]
 
 
 class Message(TypedDict, total=False):
@@ -49,13 +49,15 @@ class Message(TypedDict, total=False):
 
 
 class InputMeta(TypedDict):
-    messages: Sequence[
-        Message | dict[str, Any]  # changed to fit with tool calls
+    messages: ReadOnly[
+        Sequence[
+            Message | dict[str, object]  # changed to fit with tool calls
+        ]
     ]  # Relevant Issue: https://github.com/BerriAI/litellm/issues/9494
 
 
 class OutputMeta(TypedDict):
-    messages: Sequence[Any]
+    messages: ReadOnly[Sequence[object]]
 
 
 class DDLLMObsError(TypedDict, total=False):
@@ -71,7 +73,7 @@ class Meta(TypedDict, total=False):
     kind: Literal["llm", "tool", "task", "embedding", "retrieval"]
     input: InputMeta  # The span's input information.
     output: OutputMeta  # The span's output information.
-    metadata: dict[str, Any]
+    metadata: ReadOnly[dict[str, object]]
     error: DDLLMObsError | None  # Error information on the span
     tool_definitions: ReadOnly[Sequence[ToolDefinition]]  # The tools offered to the model on this request
 

litellm/types/llms/anthropic.py

@@ -832,6 +832,7 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
 ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01"
 
 ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18"
+ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01"
 ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01"
 
 ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14"

litellm/types/llms/openai.py

@@ -1600,7 +1600,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
 
     @field_validator("reasoning", mode="before")
     @classmethod
-    def validate_reasoning_to_dict(cls, value: Any) -> dict[str, Any] | None:
+    def validate_reasoning_to_dict(cls, value: Any) -> builtins.object:
         """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict."""
         if value is None:
             return None

litellm/types/llms/vertex_ai.py

@@ -1,6 +1,6 @@
 from collections.abc import Sequence
 from enum import Enum
-from typing import Any, Final, Literal, Protocol
+from typing import Final, Literal, Protocol
 
 from typing_extensions import (
     ReadOnly,
@@ -140,7 +140,7 @@ class Schema(TypedDict, total=False):
     title: str
     description: str
     nullable: bool
-    default: Any
+    default: ReadOnly[object]
     items: "Schema"
     minItems: str
     maxItems: str
@@ -155,7 +155,7 @@ class Schema(TypedDict, total=False):
     minLength: str
     maxLength: str
     pattern: str
-    example: Any
+    example: ReadOnly[object]
     anyOf: list["Schema"]
 
 
@@ -794,9 +794,9 @@ class VertexVideoGenerationResponse(TypedDict, total=False):
 
     name: str
     done: bool
-    response: dict[str, Any]
-    metadata: dict[str, Any]
-    error: dict[str, Any]
+    response: ReadOnly[dict[str, object]]
+    metadata: ReadOnly[dict[str, object]]
+    error: ReadOnly[dict[str, object]]
 
 
 VERTEX_CREDENTIALS_TYPES = str | dict[str, str]

litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py

@@ -48,7 +48,7 @@ class GenericGuardrailAPIMetadata(TypedDict, total=False):
 class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel):
     """Optional parameters for the Generic Guardrail API"""
 
-    additional_provider_specific_params: dict[str, Any] | None = Field(
+    additional_provider_specific_params: dict[str, object] | None = Field(
         default=None,
         description="Additional provider-specific parameters to send with the guardrail request",
     )
@@ -161,7 +161,7 @@ class GenericGuardrailAPIRequest(LiteLLMBaseModel):
         default=None,
         description="LiteLLM library version running this proxy.",
     )
-    additional_provider_specific_params: dict[str, Any] | None = None
+    additional_provider_specific_params: dict[str, object] | None = None
     tool_calls: list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] | None = None
     model: str | None = None  # the model being used for the LLM call
 

litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py

@@ -1,7 +1,5 @@
-from typing import Any
-
 from pydantic import Field
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.types.llms.base import LiteLLMBaseModel
 
@@ -14,41 +12,41 @@ from .base import IBMGuardrailsBaseConfigModel
 class IBMDetectorRequestBodyDetectorServer(TypedDict):
     """Request body for calling IBM Detector Server directly"""
 
-    contents: list[str]
-    detector_params: dict[str, Any]
+    contents: ReadOnly[list[str]]
+    detector_params: ReadOnly[dict[str, object]]
 
 
 class IBMDetectorRequestBodyOrchestrator(TypedDict):
     """Request body for calling IBM Detector via FMS Guardrails Orchestrator"""
 
-    content: str
-    detectors: dict[str, dict[str, Any]]
+    content: ReadOnly[str]
+    detectors: ReadOnly[dict[str, dict[str, object]]]
 
 
 class IBMDetectorDetection(TypedDict, total=False):
     """Individual detection from IBM Detector"""
 
-    start: int
-    end: int
-    text: str
-    detection: str
-    detection_type: str
-    score: float
-    evidences: list[Any]
-    metadata: dict[str, Any]
-    detector_id: str | None  # Only present in orchestrator response
+    start: ReadOnly[int]
+    end: ReadOnly[int]
+    text: ReadOnly[str]
+    detection: ReadOnly[str]
+    detection_type: ReadOnly[str]
+    score: ReadOnly[float]
+    evidences: ReadOnly[list[object]]
+    metadata: ReadOnly[dict[str, object]]
+    detector_id: ReadOnly[str | None]  # Only present in orchestrator response
 
 
 class IBMDetectorResponseDetectorServer(TypedDict):
     """Response from IBM Detector Server (returns list of lists)"""
 
-    detections: list[list[IBMDetectorDetection]]
+    detections: ReadOnly[list[list[IBMDetectorDetection]]]
 
 
 class IBMDetectorResponseOrchestrator(TypedDict):
     """Response from IBM FMS Guardrails Orchestrator"""
 
-    detections: list[IBMDetectorDetection]
+    detections: ReadOnly[list[IBMDetectorDetection]]
 
 
 # Pydantic Config Models
@@ -57,12 +55,12 @@ class IBMDetectorResponseOrchestrator(TypedDict):
 class IBMDetectorOptionalParams(LiteLLMBaseModel):
     """Optional parameters for IBM Detector guardrail"""
 
-    detector_params: dict[str, Any] | None = Field(
+    detector_params: dict[str, object] | None = Field(
         default_factory=lambda: {},
         description="Dictionary of arguments to pass to the detector.",
     )
 
-    extra_headers: dict[str, Any] | None = Field(
+    extra_headers: dict[str, object] | None = Field(
         default_factory=lambda: {},
         description="Dictionary of extra headers to pass to the detector.",
     )

litellm/types/proxy/management_endpoints/key_management_endpoints.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Any, Final, Literal, TypeAlias
+from typing import Final, Literal, TypeAlias
 
 from pydantic import ConfigDict, model_validator
 from typing_extensions import ReadOnly, TypedDict
@@ -50,14 +50,14 @@ class SuccessfulKeyUpdate(LiteLLMBaseModel):
     """Successfully updated key with its updated information"""
 
     key: str
-    key_info: dict[str, Any]
+    key_info: dict[str, object]
 
 
 class FailedKeyUpdate(LiteLLMBaseModel):
     """Failed key update with reason"""
 
     key: str
-    key_info: dict[str, Any] | None = None
+    key_info: dict[str, object] | None = None
     failed_reason: str
 
 
@@ -78,14 +78,14 @@ class KeyUpdateFields(LiteLLMBaseModel):
     max_budget: float | None = None
     budget_id: str | None = None
     budget_duration: str | None = None
-    budget_limits: list[Any] | None = None
-    model_max_budget: dict[str, Any] | None = None
+    budget_limits: list[object] | None = None
+    model_max_budget: dict[str, object] | None = None
 
     # Rate limits
     tpm_limit: int | None = None
     rpm_limit: int | None = None
-    model_tpm_limit: dict[str, Any] | None = None
-    model_rpm_limit: dict[str, Any] | None = None
+    model_tpm_limit: dict[str, object] | None = None
+    model_rpm_limit: dict[str, object] | None = None
     max_parallel_requests: int | None = None
     rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None
     tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None
@@ -99,7 +99,7 @@ class KeyUpdateFields(LiteLLMBaseModel):
 
     # Operational metadata
     tags: list[str] | None = None
-    metadata: dict[str, Any] | None = None
+    metadata: dict[str, object] | None = None
 
     @model_validator(mode="after")
     def validate_temp_budget(self) -> "KeyUpdateFields":

litellm/types/router.py

@@ -67,6 +67,10 @@ from .utils import (
 AllowedModelRegion: TypeAlias = Literal["eu", "us"]
 
 
+class SilentExperimentFlag(TypedDict, total=False):
+    is_silent_experiment: ReadOnly[bool]
+
+
 class FallbackSkipReason(str, enum.Enum):
     NOT_AUTHORIZED = "not_authorized"
     OVER_BUDGET = "over_budget"

litellm/types/utils.py

@@ -3374,6 +3374,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
     team_alias: str | None
     team_id: str | None
     used_client_oauth_token: ReadOnly[bool | None]
+    is_silent_experiment: ReadOnly[NotRequired[bool]]
 
 
 class AzureSpillover(TypedDict):
@@ -4130,6 +4131,18 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
         """
         return {k: v for k, v in model_info.items() if k not in cls.model_fields}
 
+    def custom_pricing_overrides(self) -> Mapping[str, object]:
+        return MappingProxyType(
+            self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
+        )
+
+    def per_second_pricing(self) -> Mapping[str, float]:
+        return MappingProxyType(
+            self.model_dump(
+                include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True
+            )
+        )
+
 
 DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
 
@@ -4599,6 +4612,7 @@ class LlmProviders(str, Enum):
     CURSOR = "cursor"
     BEDROCK_MANTLE = "bedrock_mantle"
     GDC = "gdc"
+    KYMA = "kyma"
 
 
 # Create a set of all provider values for quick lookup

provider_endpoints_support.json

@@ -5327,6 +5327,32 @@
         "generateContent": false
       }
     },
+    "kyma": {
+      "display_name": "Kyma API (`kyma`)",
+      "url": "https://docs.litellm.ai/docs/providers/kyma",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": false,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
+    },
     "bigmodel": {
       "display_name": "BigModel (Zhipu AI) (`bigmodel`)",
       "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction",

scripts/pre_commit_lint.sh

@@ -18,7 +18,8 @@
 #                      -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's
 #                         test-tree ruff and test-quality gate steps)
 #   - dashboard        -> prettier + eslint + lint budgets (test-linting.yml's ui job)
-#   - proxy/types      -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (test-linting.yml's ui-api-types job)
+#   - spec_pattern     -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift
+#                         (test-linting.yml's ui-api-types job)
 #
 # Each block is skipped when no matching files are in scope, so unrelated commits
 # stay fast. This is intentionally not auto-installed as a git hook (see
@@ -98,7 +99,7 @@ existing_files() {
 litellm_py_pattern='^litellm/.*\.py$'
 e2e_py_pattern='^tests/e2e(_harness)?/.*\.py$'
 test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|scripts/(check_test_quality|test_quality_gate)\.py)$'
-spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$'
+spec_pattern='^(litellm/.*|enterprise/.*|pyproject\.toml|uv\.lock|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$'
 ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$'
 ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$'
 
@@ -110,9 +111,6 @@ e2e_py_files=$(scope_match "$e2e_py_pattern")
 test_tree_files=$(scope_match "$test_tree_pattern")
 # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.
 fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files)
-# test-linting.yml triggers on any file under litellm/proxy or litellm/types
-# (Prisma schema and configs included, not just Python) plus the generator and its
-# lockfiles, so match that whole trigger set rather than a Python subset.
 spec_files=$(scope_match "$spec_pattern")
 # CI's ui job runs prettier over a wider extension set than eslint; keep that
 # split so this flags exactly what the job would.
@@ -337,7 +335,7 @@ summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_fi
 summary_item "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_files" \
     "no tests/ Python files or test-tree lint inputs in scope"
 summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope"
-summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope"
+summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/, enterprise/, dependency, or generator files in scope"
 
 if [ -z "$litellm_py_files$e2e_py_files$test_tree_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then
     echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2

tests/integration/_support/client.py

@@ -280,12 +280,14 @@ class Scenario:
             self.gateway.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == []
         )
 
-    def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str:
-        name: Final = f"integration-{uuid.uuid4().hex}"
+    def model(
+        self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue
+    ) -> str:
+        model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}"
         created: Final = self.gateway.post(
             "/model/new",
             {
-                "model_name": name,
+                "model_name": model_name,
                 "litellm_params": {
                     "model": "openai/gpt-4o-mini",
                     "api_key": "integration-provider-key",
@@ -298,8 +300,8 @@ class Scenario:
         identity: Final = string_value(object_value(created["model_info"])["id"])
         self.cleanups.callback(self.delete_model, identity)
         if self.gateway.workers > 1:
-            self._await_every_worker(name, identity)
-        return name
+            self._await_every_worker(model_name, identity)
+        return model_name
 
     def _await_every_worker(self, name: str, identity: str) -> None:
         from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS

tests/integration/caching/test_response_cache_model_group_key.py

@@ -0,0 +1,192 @@
+"""The response cache keys an answer by the router's model group on every endpoint the router serves.
+
+The router names the model group in ``metadata`` on ``/v1/chat/completions`` and in ``litellm_metadata`` on
+``/v1/responses`` and ``/v1/messages``. Every cell gives each model group its own scripted upstream scenario on one
+shared provider model, fills one group's entry, and reads which scenario the identical request reached next and which
+answer the caller got: an answer served from the cache repeats the id of the request that filled it.
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, TypeAlias
+
+import pytest
+import yaml
+from anthropic import Anthropic
+from openai import OpenAI
+from pydantic import JsonValue
+
+from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, object_value
+from tests.integration._support.process import OwnedState, graceful_stop_seconds, owned_proxy_process
+from tests.integration._support.upstream import ScenarioHandle
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    MESSAGE,
+    MESSAGES_MODEL,
+    PROVIDER_KEY,
+    RESPONSE,
+    Upstream,
+    await_entry,
+    json_response,
+    prompt,
+    redis_store,
+    scripted,
+)
+
+Ask: TypeAlias = Callable[[Gateway, str, str, Mapping[str, JsonValue]], str]
+
+
+def _root(gateway: Gateway) -> str:
+    return str(gateway.client.base_url).rstrip("/")
+
+
+def _metadata(metadata: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None:
+    return {"metadata": dict(metadata)} if metadata else None
+
+
+def _chat(gateway: Gateway, model: str, text: str, metadata: Mapping[str, JsonValue]) -> str:
+    with OpenAI(base_url=f"{_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client:
+        return client.chat.completions.create(
+            model=model, messages=[{"role": "user", "content": text}], extra_body=_metadata(metadata)
+        ).id
+
+
+def _responses(gateway: Gateway, model: str, text: str, metadata: Mapping[str, JsonValue]) -> str:
+    with OpenAI(base_url=f"{_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client:
+        identity: Final = client.responses.create(model=model, input=text, extra_body=_metadata(metadata)).output[0].id
+    assert identity is not None
+    return identity
+
+
+def _messages(gateway: Gateway, model: str, text: str, metadata: Mapping[str, JsonValue]) -> str:
+    with Anthropic(base_url=_root(gateway), api_key=gateway.key, max_retries=0) as client:
+        return client.messages.create(
+            model=model,
+            max_tokens=16,
+            messages=[{"role": "user", "content": text}],
+            extra_body=_metadata(metadata),
+        ).id
+
+
+@dataclass(frozen=True, slots=True)
+class Endpoint:
+    provider_model: str
+    sibling_model: str
+    answer: Mapping[str, JsonValue]
+    ask: Ask
+
+
+@dataclass(frozen=True, slots=True)
+class Group:
+    name: str
+    upstream: ScenarioHandle
+
+
+ENDPOINTS: Final = {
+    "chat_completions": Endpoint(CHAT_MODEL, "openai/gpt-5.6", CHAT, _chat),
+    "responses": Endpoint(CHAT_MODEL, "openai/gpt-5.6", RESPONSE, _responses),
+    "messages": Endpoint(MESSAGES_MODEL, "anthropic/claude-sonnet-5-5", MESSAGE, _messages),
+}
+
+
+def _group(scenario: Scenario, endpoint: Endpoint, name: str | None = None) -> Group:
+    handle: Final = scripted(scenario, json_response(endpoint.answer))
+    return Group(scenario.model(name=name, model=endpoint.provider_model, api_base=handle.api_base()), handle)
+
+
+@pytest.mark.parametrize("endpoint", ENDPOINTS.values(), ids=ENDPOINTS.keys())
+def test_identical_requests_to_two_groups_on_one_model_each_reach_their_own_deployment(
+    gateway: Gateway, endpoint: Endpoint
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        first: Final = _group(scenario, endpoint)
+        second: Final = _group(scenario, endpoint)
+        filled: Final = endpoint.ask(gateway, first.name, text, {})
+        await_entry(store, filled)
+        answered: Final = endpoint.ask(gateway, second.name, text, {})
+        assert answered != filled, f"{second.name} was served the answer cached for {first.name}"
+        assert upstream.calls(second.upstream.scenario_id) == 1
+        assert endpoint.ask(gateway, first.name, text, {}) == filled
+        assert upstream.calls(first.upstream.scenario_id) == 1
+
+
+PLANTS: Final[Mapping[str, Callable[[str], Mapping[str, JsonValue]]]] = {
+    "model_group": lambda target: {"model_group": target},
+    "caching_groups": lambda target: {"caching_groups": [target]},
+}
+
+
+@pytest.mark.parametrize("plant", PLANTS.values(), ids=PLANTS.keys())
+@pytest.mark.parametrize("endpoint", ENDPOINTS.values(), ids=ENDPOINTS.keys())
+def test_a_caller_naming_another_group_in_metadata_never_fills_that_groups_entry(
+    gateway: Gateway, endpoint: Endpoint, plant: Callable[[str], Mapping[str, JsonValue]]
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        caller: Final = _group(scenario, endpoint)
+        target: Final = _group(scenario, endpoint, name=f"{caller.name}-premium")
+        planted: Final = endpoint.ask(gateway, caller.name, text, plant(target.name))
+        assert upstream.calls(caller.upstream.scenario_id) == 1
+        await_entry(store, planted)
+        answered: Final = endpoint.ask(gateway, target.name, text, {})
+        assert answered != planted, f"{target.name} was served the answer {caller.name} planted under its name"
+        assert upstream.calls(target.upstream.scenario_id) == 1
+        await_entry(store, answered)
+        assert endpoint.ask(gateway, target.name, text, {}) == answered
+
+
+def _caching_groups_config(directory: Path, endpoint: Endpoint, first: Group, second: Group) -> Path:
+    rig: Final = JSON_OBJECT.validate_python(
+        yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text())
+    )
+    router_settings: Final = object_value(rig.get("router_settings") or {})
+    config: Final = {
+        **rig,
+        "model_list": [
+            {
+                "model_name": group.name,
+                "litellm_params": {"model": model, "api_key": PROVIDER_KEY, "api_base": group.upstream.api_base()},
+            }
+            for group, model in ((first, endpoint.provider_model), (second, endpoint.sibling_model))
+        ],
+        "router_settings": {**router_settings, "caching_groups": [[first.name, second.name]]},
+    }
+    path: Final = directory / f"proxy_config-{uuid.uuid4().hex[:8]}.yaml"
+    path.write_text(yaml.safe_dump(config, sort_keys=False))
+    return path
+
+
+def _configured_group(scenario: Scenario, endpoint: Endpoint) -> Group:
+    return Group(f"cache-group-{uuid.uuid4().hex[:8]}", scripted(scenario, json_response(endpoint.answer)))
+
+
+@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+@pytest.mark.parametrize("endpoint", ENDPOINTS.values(), ids=ENDPOINTS.keys())
+def test_groups_in_one_caching_group_on_different_models_share_one_entry(
+    gateway: Gateway, endpoint: Endpoint, tmp_path: Path
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        first: Final = _configured_group(scenario, endpoint)
+        second: Final = _configured_group(scenario, endpoint)
+        config: Final = _caching_groups_config(tmp_path, endpoint, first, second)
+        with owned_proxy_process(
+            gateway, tmp_path, {}, config=config, workers=1, state=OwnedState.SHARED_WITH_WORKER
+        ) as owned:
+            filled: Final = endpoint.ask(owned.gateway, first.name, text, {})
+            await_entry(store, filled)
+            assert endpoint.ask(owned.gateway, second.name, text, {}) == filled
+            assert upstream.calls(second.upstream.scenario_id) == 0
+            assert upstream.calls(first.upstream.scenario_id) == 1

tests/integration/management/test_fallback_management_public_names.py

@@ -1,6 +1,7 @@
 import uuid
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
 from functools import partial
 from itertools import chain
 from pathlib import Path
@@ -10,6 +11,7 @@ import httpx
 import psutil
 import pytest
 from pydantic import JsonValue
+from redis import Redis
 
 from tests.integration._support.client import (
     Gateway,
@@ -27,7 +29,7 @@ from tests.integration._support.process import (
     setup_only_proxy_run,
     worker_configuration,
 )
-from tests.integration._support.redis_process import owned_redis
+from tests.integration._support.redis_process import OwnedRedis, owned_redis
 
 pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
 
@@ -385,6 +387,39 @@ def test_fallback_write_survives_a_redis_outage(gateway: Gateway, tmp_path: Path
                 cache.start()
 
 
+@contextmanager
+def _refusing_cache_deletes(cache: OwnedRedis) -> Iterator[None]:
+    with Redis(host=cache.host, port=cache.port) as client:
+        client.acl_setuser("default", enabled=True, commands=["-del", "-unlink"])
+        try:
+            yield
+        finally:
+            client.acl_setuser("default", enabled=True, commands=["+del", "+unlink"])
+
+
+@pytest.mark.usefixtures("migrated_database")
+def test_consecutive_creates_keep_every_rule_when_redis_refuses_the_cache_eviction(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    with owned_redis(tmp_path) as cache:
+        overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)}
+        with owned_proxy_process(gateway, tmp_path, overrides, database_setup=()) as owned:
+            with owned.gateway.scenario() as scenario:
+                target: Final = scenario.model(model=f"openai/x3t-{uuid.uuid4().hex}")
+                first: Final = scenario.model(model=f"openai/x3a-{uuid.uuid4().hex}")
+                second: Final = scenario.model(model=f"openai/x3b-{uuid.uuid4().hex}")
+                for model in (target, first, second):
+                    _wait_until_served(owned.gateway, model)
+                with _refusing_cache_deletes(cache):
+                    created: Final = tuple(
+                        _create_fallback(owned.gateway, scenario, model, [target]) for model in (first, second)
+                    )
+                    assert all(response.status_code == 200 for response in created), [r.text for r in created]
+                    covered: Final = _covered_models(_stored_fallbacks())
+                    assert {first, second} <= covered, covered
+                    assert EVICTION_WARNING in owned.log.read_text(), owned.log.read_text()[-3000:]
+
+
 @pytest.mark.usefixtures("migrated_database")
 def test_fallback_write_survives_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None:
     with owned_proxy_process(gateway, tmp_path, {}, workers=2, database_setup=()) as owned:

tests/integration/messages_endpoint/test_beta_gated_fields_wire.py

@@ -0,0 +1,183 @@
+import json
+import re
+import uuid
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+import pytest
+from integration._support.client import Gateway, gateway_from_environment, list_value, object_value
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.shares_worker_state
+
+ANTHROPIC_MODEL: Final = "beta-gated-anthropic"
+FOUNDRY_MODEL: Final = "beta-gated-foundry"
+FOUNDRY_DROP_PARAMS_MODEL: Final = "beta-gated-foundry-drop-params"
+CLEAR_AT_BETA: Final = "mid-conversation-system-clear-at-2026-08-21"
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_NONCE: Final = re.compile(rb"nonce-([0-9a-f]{32})")
+
+
+def nonce() -> str:
+    return f"nonce-{uuid.uuid4().hex}"
+
+
+def claude_reply(request: Request) -> Reply:
+    match: Final = _NONCE.search(request.body)
+    assert match is not None, request.body[:200]
+    identity: Final = match.group(1).decode()
+    return Reply(
+        body=json.dumps(
+            {
+                "id": f"msg_{identity}",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-opus-4-8",
+                "content": [{"type": "text", "text": f"answered {identity}"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 12, "output_tokens": 3},
+            }
+        ).encode()
+    )
+
+
+def respond(request: Request) -> Reply:
+    if request.method != "POST":
+        return Reply(body=b'{"object":"list","data":[]}')
+    return claude_reply(request)
+
+
+def deployment(name: str, model: str, api_base: str, drop_params: bool = False) -> dict[str, JsonValue]:
+    return {
+        "model_name": name,
+        "litellm_params": {
+            "model": model,
+            "api_base": api_base,
+            "api_key": f"synthetic-{name}-key",
+            **({"drop_params": True} if drop_params else {}),
+        },
+    }
+
+
+def write_config(directory: Path, wire_url: str) -> Path:
+    path: Final = directory / f"beta-gated-{uuid.uuid4().hex}.json"
+    path.write_text(
+        json.dumps(
+            {
+                "model_list": [
+                    deployment(ANTHROPIC_MODEL, "anthropic/claude-opus-4-8", f"{wire_url}/anthropic"),
+                    deployment(FOUNDRY_MODEL, "azure_ai/claude-opus-4-8", f"{wire_url}/foundry"),
+                    deployment(
+                        FOUNDRY_DROP_PARAMS_MODEL, "azure_ai/claude-opus-4-8", f"{wire_url}/foundry", drop_params=True
+                    ),
+                ],
+                "general_settings": {
+                    "master_key": "os.environ/LITELLM_MASTER_KEY",
+                    "database_url": "os.environ/DATABASE_URL",
+                },
+            }
+        )
+    )
+    return path
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    gateway: Gateway
+    wire: Wire
+
+    def send(self, model: str, request_nonce: str) -> tuple[int, bytes]:
+        response: Final = self.gateway.client.post(
+            "/v1/messages",
+            json={
+                "model": model,
+                "max_tokens": 64,
+                "messages": [
+                    {"role": "user", "content": f"hi {request_nonce}"},
+                    {"role": "assistant", "content": "Hello"},
+                    {"role": "user", "content": "how are you"},
+                    {"role": "system", "content": "Answer in French", "clear_at": "next_user_message"},
+                ],
+            },
+            headers={"Authorization": f"Bearer {self.gateway.key}", "anthropic-beta": CLEAR_AT_BETA},
+            timeout=120,
+        )
+        return response.status_code, response.content
+
+    def received(self, request_nonce: str) -> Request:
+        requests: Final = tuple(request for request in self.wire.drain() if request_nonce.encode() in request.body)
+        assert len(requests) == 1, [request.target for request in requests]
+        return requests[0]
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    directory: Final = tmp_path_factory.mktemp("beta-gated")
+    with (
+        gateway_from_environment() as shared,
+        wire_server(respond) as wire,
+        owned_proxy(
+            shared,
+            directory,
+            {"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS": "true"},
+            config=write_config(directory, wire.url),
+            workers=1,
+        ) as gateway,
+    ):
+        yield Rig(gateway, wire)
+
+
+def reply_text(status: int, raw: bytes) -> str:
+    assert status == 200, raw[:500]
+    return "".join(str(object_value(block)["text"]) for block in list_value(_JSON_OBJECT.validate_json(raw)["content"]))
+
+
+def beta_header(headers: Mapping[str, str]) -> str:
+    return ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta")
+
+
+def system_message(body: bytes) -> dict[str, JsonValue]:
+    messages: Final = list_value(_JSON_OBJECT.validate_json(body)["messages"])
+    system: Final = [object_value(m) for m in messages if object_value(m).get("role") == "system"]
+    assert len(system) == 1, messages
+    return system[0]
+
+
+def test_a_drop_params_leg_whose_row_lacks_the_beta_sends_neither_the_beta_nor_clear_at(rig: Rig) -> None:
+    request_nonce: Final = nonce()
+    status, raw = rig.send(FOUNDRY_DROP_PARAMS_MODEL, request_nonce)
+    assert reply_text(status, raw) == f"answered {request_nonce.removeprefix('nonce-')}"
+    upstream: Final = rig.received(request_nonce)
+    assert CLEAR_AT_BETA not in beta_header(upstream.headers), upstream.headers
+    assert system_message(upstream.body) == {"role": "system", "content": "Answer in French"}
+
+
+def test_a_leg_whose_row_lacks_the_beta_forwards_clear_at_as_sent_without_drop_params(rig: Rig) -> None:
+    request_nonce: Final = nonce()
+    status, raw = rig.send(FOUNDRY_MODEL, request_nonce)
+    assert reply_text(status, raw) == f"answered {request_nonce.removeprefix('nonce-')}"
+    upstream: Final = rig.received(request_nonce)
+    assert CLEAR_AT_BETA not in beta_header(upstream.headers), upstream.headers
+    assert system_message(upstream.body) == {
+        "role": "system",
+        "content": "Answer in French",
+        "clear_at": "next_user_message",
+    }
+
+
+def test_a_leg_that_carries_the_beta_forwards_clear_at_untouched(rig: Rig) -> None:
+    request_nonce: Final = nonce()
+    status, raw = rig.send(ANTHROPIC_MODEL, request_nonce)
+    assert reply_text(status, raw) == f"answered {request_nonce.removeprefix('nonce-')}"
+    upstream: Final = rig.received(request_nonce)
+    assert CLEAR_AT_BETA in beta_header(upstream.headers), upstream.headers
+    assert system_message(upstream.body) == {
+        "role": "system",
+        "content": "Answer in French",
+        "clear_at": "next_user_message",
+    }

tests/integration/messages_endpoint/test_guardrail_block_stream.py

@@ -0,0 +1,129 @@
+import uuid
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+import anthropic
+import httpx
+import pytest
+import yaml
+from anthropic.types import Message
+from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment
+from integration._support.process import graceful_stop_seconds, owned_proxy
+from integration._support.wire import Reply, Request, wire_server
+
+pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+
+_DENIAL: Final = "This request is not allowed by policy."
+_CALL_ID_HEADER: Final = "x-litellm-call-id"
+
+
+@dataclass(frozen=True, slots=True)
+class _Rail:
+    gateway: Gateway
+    names: tuple[str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class _Ask:
+    model: str
+    rails: tuple[str, str]
+    call_id: str
+
+
+@pytest.fixture(scope="module")
+def denying_rail(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rail]:
+    directory: Final = tmp_path_factory.mktemp("messages-guardrail-block")
+    name: Final = f"messages-deny-{uuid.uuid4().hex}"
+    post_call_name: Final = f"messages-deny-replies-{uuid.uuid4().hex}"
+    stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final = directory / "messages-guardrail-block.yaml"
+    config.write_text(
+        yaml.safe_dump(
+            {
+                **stock,
+                "guardrails": [
+                    {
+                        "guardrail_name": name,
+                        "litellm_params": {
+                            "guardrail": "custom_code",
+                            "mode": "pre_call",
+                            "default_on": False,
+                            "custom_code": (
+                                f"def apply_guardrail(inputs, request_data, input_type):\n    return block({_DENIAL!r})\n"
+                            ),
+                        },
+                    },
+                    {
+                        "guardrail_name": post_call_name,
+                        "litellm_params": {
+                            "guardrail": "custom_code",
+                            "mode": "post_call",
+                            "default_on": False,
+                            "custom_code": (
+                                "def apply_guardrail(inputs, request_data, input_type):\n"
+                                "    return block('A post-call rail scanned the block message.')\n"
+                            ),
+                        },
+                    },
+                ],
+            }
+        )
+    )
+    with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as candidate:
+        yield _Rail(candidate, (name, post_call_name))
+
+
+def _provider_must_not_answer(request: Request) -> Reply:
+    return Reply(status=500, body=f"unexpected {request.method} {request.target}".encode())
+
+
+def _streamed(client: anthropic.Anthropic, ask: _Ask) -> tuple[Message, httpx.Headers]:
+    with client.messages.stream(
+        model=ask.model,
+        max_tokens=32,
+        messages=[{"role": "user", "content": "count from 1 to 100"}],
+        extra_body={"guardrails": list(ask.rails)},
+        extra_headers={_CALL_ID_HEADER: ask.call_id},
+    ) as stream:
+        return stream.get_final_message(), stream.response.headers
+
+
+def _created(client: anthropic.Anthropic, ask: _Ask) -> tuple[Message, httpx.Headers]:
+    raw: Final = client.messages.with_raw_response.create(
+        model=ask.model,
+        max_tokens=32,
+        messages=[{"role": "user", "content": "count from 1 to 100"}],
+        extra_body={"guardrails": list(ask.rails)},
+        extra_headers={_CALL_ID_HEADER: ask.call_id},
+    )
+    return raw.parse(), raw.headers
+
+
+@pytest.mark.parametrize("send", (_streamed, _created), ids=("stream", "json"))
+def test_anthropic_sdk_reads_a_pre_call_block_as_a_finished_message_past_post_call_rails(
+    denying_rail: _Rail, send: Callable[[anthropic.Anthropic, _Ask], tuple[Message, httpx.Headers]]
+) -> None:
+    call_id: Final = f"messages-block-{uuid.uuid4().hex}"
+    with (
+        wire_server(_provider_must_not_answer) as provider,
+        denying_rail.gateway.scenario() as scenario,
+        httpx.Client(timeout=30, trust_env=False) as transport,
+        anthropic.Anthropic(
+            api_key=denying_rail.gateway.key,
+            base_url=str(denying_rail.gateway.client.base_url).rstrip("/"),
+            max_retries=0,
+            http_client=transport,
+        ) as client,
+    ):
+        model: Final = scenario.model(
+            model="anthropic/claude-haiku-5-5", api_base=provider.url, api_key="synthetic-anthropic-key"
+        )
+        message, headers = send(client, _Ask(model, denying_rail.names, call_id))
+        sent: Final = provider.drain()
+
+    assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _DENIAL)], message
+    assert message.stop_reason == "end_turn", message
+    assert headers.get(_CALL_ID_HEADER) == call_id, dict(headers)
+    assert sent == (), sent

tests/integration/observability/test_guardrail_effects.py

@@ -1902,11 +1902,20 @@ def test_messages_pre_call_denial_streams_message(gateway: Gateway, tmp_path: Pa
         )
         assert response.status_code == 200, response.text
         assert response.headers["content-type"].startswith("text/event-stream"), response.text
-        lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: "))
-        assert len(lines) == 1, response.text
-        body: Final = json.loads(lines[0].removeprefix("data: "))
-        assert body["type"] == "message" and body["role"] == "assistant", body
-        assert body["content"] == [{"type": "text", "text": _RESPONSES_DENIAL}], body
+        events: Final = tuple(
+            json.loads(line.removeprefix("data: ")) for line in response.text.split("\n") if line.startswith("data: ")
+        )
+        assert [event["type"] for event in events] == [
+            "message_start",
+            "content_block_start",
+            "content_block_delta",
+            "content_block_stop",
+            "message_delta",
+            "message_stop",
+        ], response.text
+        assert events[0]["message"]["role"] == "assistant", events[0]
+        assert events[2]["delta"] == {"type": "text_delta", "text": _RESPONSES_DENIAL}, events[2]
+        assert events[4]["delta"]["stop_reason"] == "end_turn", events[4]
 
 
 @pytest.mark.covers("other.observability.guardrails.responses_pre_call_denial_writes_zero_spend_row")

tests/integration/sdk/test_router_silent_model_shadow_sdk.py

@@ -82,9 +82,9 @@ class _ShadowRecorder(CustomLogger):
     async def async_log_success_event(
         self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
     ) -> None:
-        metadata: Final = _OBJECT.validate_python(_OBJECT.validate_python(kwargs["litellm_params"])["metadata"] or {})
-        if metadata.get("is_silent_experiment") is True:
-            self.shadow_groups.put(str(metadata.get("model_group")))
+        payload: Final = _OBJECT.validate_python(kwargs["standard_logging_object"])
+        if _OBJECT.validate_python(payload["metadata"]).get("is_silent_experiment") is True:
+            self.shadow_groups.put(str(payload["model_group"]))
 
     def drained(self) -> tuple[str, ...]:
         return tuple(self.shadow_groups.get_nowait() for _ in range(self.shadow_groups.qsize()))

tests/unit/caching/test_unit_test_caching.py

@@ -8,6 +8,7 @@ load_dotenv()
 import asyncio
 import hashlib
 import random
+from typing import Final
 
 import pytest
 
@@ -16,7 +17,7 @@ from litellm import aembedding, completion, embedding
 from litellm.caching.caching import Cache
 
 from unittest.mock import AsyncMock, patch, MagicMock
-from litellm.caching.caching_handler import LLMCachingHandler, CachingHandlerResponse
+from litellm.caching.caching_handler import LLMCachingHandler, CachingHandlerResponse, _PENDING_CACHE_WRITES
 from litellm.caching.caching import LiteLLMCacheType
 from litellm.types.utils import CallTypes
 from litellm.types.rerank import RerankResponse
@@ -278,6 +279,107 @@ def test_get_model_param_value_reads_model_group_from_litellm_metadata():
     )
 
 
+def test_get_model_param_value_prefers_the_router_group_over_a_caller_model_group():
+    cache: Final = Cache()
+    deployment: Final = {"id": "deployment-1"}
+    request: Final = {"model": "openai/gpt-5.6", "input": "search this text", "model_info": deployment}
+    router_metadata: Final = {"model_group": "caller", "model_info": deployment}
+
+    assert (
+        cache._get_model_param_value(
+            {**request, "metadata": {"model_group": "target"}, "litellm_metadata": router_metadata}
+        )
+        == "caller"
+    )
+    assert (
+        cache._get_model_param_value(
+            {
+                **request,
+                "litellm_params": {"metadata": {"model_group": "target"}, "litellm_metadata": router_metadata},
+            }
+        )
+        == "caller"
+    )
+    assert (
+        cache._get_model_param_value(
+            {
+                **request,
+                "metadata": {"model_group": "target", "model_info": deployment},
+                "litellm_metadata": router_metadata,
+            }
+        )
+        == "caller"
+    )
+    assert (
+        cache._get_model_param_value(
+            {
+                **request,
+                "metadata": router_metadata,
+                "litellm_metadata": {"model_group": "parent", "model_info": {"id": "parent-deployment"}},
+            }
+        )
+        == "caller"
+    )
+
+
+def test_get_model_param_value_keeps_metadata_first_outside_the_router():
+    cache: Final = Cache()
+
+    assert (
+        cache._get_model_param_value(
+            {
+                "model": "openai/gpt-5.6",
+                "input": "search this text",
+                "metadata": {"model_group": "group-a"},
+                "litellm_metadata": {"model_group": "group-b"},
+            }
+        )
+        == "group-a"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_call_keeps_its_own_group_when_forwarded_litellm_metadata_names_another(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL))
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": group,
+                "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test", "mock_response": f"{group} answer"},
+            }
+            for group in ("group-a", "group-b")
+        ]
+    )
+    messages: Final = [{"role": "user", "content": "which group answers this"}]
+
+    await router.acompletion(
+        model="group-a",
+        messages=messages,
+        caching=True,
+        litellm_metadata={"model_group": "group-b", "model_info": {"id": "parent-deployment"}},
+    )
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+    reply: Final = await router.acompletion(model="group-b", messages=messages, caching=True)
+
+    assert reply.choices[0].message.content == "group-b answer"
+
+
+@pytest.mark.parametrize("planted", [["sales-premium"], [("sales", "sales-premium")]], ids=["string", "tuple"])
+def test_get_model_param_value_reads_caching_groups_only_beside_the_router_group(planted: list[str | tuple[str, str]]):
+    cache: Final = Cache()
+    request: Final = {
+        "model": "openai/gpt-5.6",
+        "input": "search this text",
+        "model_info": {"id": "sales-deployment"},
+        "litellm_metadata": {"model_group": "sales", "caching_groups": None, "model_info": {"id": "sales-deployment"}},
+        "metadata": {"caching_groups": planted},
+    }
+
+    assert cache._get_model_param_value(request) == "sales"
+
+
 def test_preset_cache_key():
     """
     Test that the preset cache key is used if it is set in kwargs["litellm_params"]

tests/unit/integrations/datadog/test_datadog_cost_management.py

@@ -1,4 +1,5 @@
 import time
+from typing import Final
 from unittest.mock import AsyncMock
 
 import pytest
@@ -314,6 +315,43 @@ async def test_extract_tags_nested_metadata_allowlisted(clean_env):
     assert "ignored" not in tags
 
 
+@pytest.mark.parametrize(
+    ("metadata", "request_tags", "expected_tag"),
+    [
+        (
+            {"is_silent_experiment": True, "requester_metadata": {"is_silent_experiment": False}},
+            ["is_silent_experiment:false"],
+            "true",
+        ),
+        (
+            {
+                "requester_metadata": {"is_silent_experiment": True},
+                "spend_logs_metadata": {"is_silent_experiment": "true"},
+            },
+            [],
+            None,
+        ),
+        ({}, ["is_silent_experiment:true"], None),
+    ],
+    ids=["mirrored call", "direct call with a client flag", "direct call with a client request tag"],
+)
+@pytest.mark.asyncio
+async def test_extract_tags_takes_the_silent_experiment_flag_only_from_the_router(
+    clean_env: None, metadata: dict[str, object], request_tags: list[str], expected_tag: str | None
+):
+    logger: Final = DatadogCostManagementLogger(cost_tag_keys=["is_silent_experiment"])
+    log: Final = StandardLoggingPayload(
+        custom_llm_provider="openai",
+        model="gpt-4",
+        response_cost=0.01,
+        startTime=time.time(),
+        metadata=metadata,
+        request_tags=request_tags,
+    )
+
+    assert logger._extract_tags(log).get("is_silent_experiment") == expected_tag
+
+
 @pytest.mark.asyncio
 async def test_extract_tags_allowlist_cannot_override_reserved_dimensions(clean_env):
     """

tests/unit/integrations/test_langfuse.py

@@ -830,6 +830,31 @@ class TestLangfuseUsageDetails(unittest.TestCase):
         assert generation_metadata["litellm_response_cost"] == 0.25
         assert generation_metadata["api_base"] == "https://real-api-base"
 
+    def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self):
+        payload = self._build_standard_logging_payload(trace_id="canary-trace-id")
+        payload["metadata"]["is_silent_experiment"] = True
+        payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False}
+        kwargs = self._build_langfuse_kwargs(payload)
+        metadata = self._canary_request_metadata()
+        self.use_real_langfuse_client()
+
+        self.logger._log_langfuse_v2(
+            user_id="user-1",
+            metadata=metadata,
+            litellm_params={"metadata": metadata},
+            output=None,
+            start_time=datetime.datetime(2024, 1, 1, 12, 0, 0),
+            end_time=datetime.datetime(2024, 1, 1, 12, 0, 1),
+            kwargs=kwargs,
+            optional_params={},
+            input=None,
+            response_obj=None,
+            level="INFO",
+            litellm_call_id="canary-call-id",
+        )
+
+        assert self.exported_generation_metadata()["is_silent_experiment"] is True
+
     def test_generation_metadata_carries_the_call_id_and_response_id(self):
         """
         v2's generation id was ``time-<hh-mm-ss-us>_<response id>``, so a generation could

tests/unit/integrations/test_prometheus_spend_logs_metadata.py

@@ -5,6 +5,10 @@ Verifies that metadata from x-litellm-spend-logs-metadata header is available
 in Prometheus custom labels via combined_metadata.
 """
 
+from typing import Final
+
+import pytest
+
 from litellm.integrations.prometheus import (
     _get_combined_custom_metadata_from_standard_logging_payload,
     get_custom_labels_from_metadata,
@@ -205,3 +209,34 @@ def test_project_alias_accessible_without_prefix(monkeypatch):
     result = get_custom_labels_from_metadata(combined)
 
     assert result == {"user_api_key_project_alias": "hotel-recommendations"}
+
+
+@pytest.mark.parametrize(
+    ("standard_logging_metadata", "expected_labels"),
+    [
+        (
+            {"is_silent_experiment": True, "requester_metadata": {"is_silent_experiment": False}},
+            {"metadata_is_silent_experiment": "true"},
+        ),
+        (
+            {
+                "requester_metadata": {"is_silent_experiment": True},
+                "spend_logs_metadata": {"is_silent_experiment": "true"},
+            },
+            {},
+        ),
+    ],
+    ids=["mirrored call", "direct call with a client flag"],
+)
+def test_silent_experiment_label_comes_only_from_the_router_flag(
+    monkeypatch: pytest.MonkeyPatch,
+    standard_logging_metadata: dict[str, object],
+    expected_labels: dict[str, str],
+):
+    monkeypatch.setattr("litellm.custom_prometheus_metadata_labels", ["metadata.is_silent_experiment"])
+
+    combined: Final = _get_combined_custom_metadata_from_standard_logging_payload(
+        {"metadata": standard_logging_metadata}
+    )
+
+    assert get_custom_labels_from_metadata(combined) == expected_labels

tests/unit/litellm_core_utils/test_litellm_logging.py

@@ -50,6 +50,7 @@ from litellm.litellm_core_utils.litellm_logging import (
     get_standard_logging_object_payload,
     set_callbacks,
 )
+from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
 from litellm.proxy._types import UserAPIKeyAuth
@@ -1118,6 +1119,115 @@ class TestZeroCostDiagnostic:
             litellm.model_cost.pop(served_model, None)
 
 
+class TestPerSecondMessagesCost:
+    DEPLOYMENT_ID: Final = "per-second-priced-deployment"
+    MODEL_GROUP: Final = "per-second-priced-claude"
+    RATE: Final = 0.00042
+    PER_SECOND_PRICING: Final = {"input_cost_per_second": RATE, "output_cost_per_second": RATE}
+    MESSAGES_RESPONSE: Final = {
+        "id": "msg_per-second-call",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude-haiku-5-5",
+        "content": [{"type": "text", "text": "Hi there friend"}],
+        "stop_reason": "end_turn",
+        "stop_sequence": None,
+        "usage": {"input_tokens": 10, "output_tokens": 5},
+    }
+
+    @pytest.fixture(autouse=True)
+    def registered_deployment(self) -> Iterator[None]:
+        litellm.register_model(model_cost={self.DEPLOYMENT_ID: self.PER_SECOND_PRICING}, persist_across_reloads=False)
+        try:
+            yield
+        finally:
+            litellm.model_cost.pop(self.DEPLOYMENT_ID, None)
+
+    def _routed_messages_call(
+        self, *, stream: bool, received_seconds_before_start: float | None = None
+    ) -> LitellmLogging:
+        started: Final = datetime.datetime.now()
+        proxy_receive_stamp: Final = (
+            {}
+            if received_seconds_before_start is None
+            else {
+                "litellm_received_at": (started - datetime.timedelta(seconds=received_seconds_before_start)).astimezone(
+                    datetime.timezone.utc
+                )
+            }
+        )
+        logging_obj: Final = LitellmLogging(
+            model=self.MODEL_GROUP,
+            messages=[{"role": "user", "content": "Say hi in three words"}],
+            stream=stream,
+            call_type="anthropic_messages",
+            start_time=started,
+            litellm_call_id="per-second-call",
+            function_id="fn",
+            kwargs={"model": self.MODEL_GROUP},
+        )
+        logging_obj.update_from_kwargs(
+            kwargs={
+                "model": "anthropic/claude-haiku-5-5",
+                "litellm_metadata": {
+                    "deployment": "anthropic/claude-haiku-5-5",
+                    "model_info": {"id": self.DEPLOYMENT_ID},
+                    **proxy_receive_stamp,
+                },
+            },
+            model="claude-haiku-5-5",
+            optional_params={"max_tokens": 32},
+            litellm_params={
+                "preset_cache_key": None,
+                "stream_response": {},
+                "model_info": None,
+                **self.PER_SECOND_PRICING,
+                "max_tokens": 32,
+            },
+            custom_llm_provider="anthropic",
+        )
+        return logging_obj
+
+    @staticmethod
+    def _response_handed_back_after(logging_obj: LitellmLogging, seconds: float) -> None:
+        started: Final = logging_obj.start_time
+        update_response_metadata(
+            result=dict(TestPerSecondMessagesCost.MESSAGES_RESPONSE),
+            logging_obj=logging_obj,
+            model="claude-haiku-5-5",
+            kwargs={},
+            start_time=started,
+            end_time=started + datetime.timedelta(seconds=seconds),
+        )
+
+    def test_messages_call_prices_the_reported_response_time_however_late_the_cost_is_read(self) -> None:
+        logging_obj: Final = self._routed_messages_call(stream=False)
+        self._response_handed_back_after(logging_obj, seconds=1.0)
+        result: Final = logging_obj._handle_anthropic_messages_response_logging(result=dict(self.MESSAGES_RESPONSE))
+        cost_for_the_response_header: Final = logging_obj._response_cost_calculator(result=result)
+        logging_obj.model_call_details["end_time"] = logging_obj.start_time + datetime.timedelta(seconds=1.5)
+        cost_for_the_spend_log: Final = logging_obj._response_cost_calculator(result=result)
+
+        assert cost_for_the_response_header == pytest.approx(self.RATE)
+        assert cost_for_the_spend_log == pytest.approx(self.RATE)
+
+    def test_messages_call_bills_from_logging_setup_not_from_proxy_receive(self) -> None:
+        logging_obj: Final = self._routed_messages_call(stream=False, received_seconds_before_start=2.0)
+        self._response_handed_back_after(logging_obj, seconds=1.0)
+        result: Final = logging_obj._handle_anthropic_messages_response_logging(result=dict(self.MESSAGES_RESPONSE))
+
+        assert logging_obj.response_timing_metrics["_response_ms"] == pytest.approx(3000)
+        assert logging_obj._response_cost_calculator(result=result) == pytest.approx(self.RATE)
+
+    def test_streamed_messages_call_bills_the_whole_stream_not_the_first_byte(self) -> None:
+        logging_obj: Final = self._routed_messages_call(stream=True)
+        self._response_handed_back_after(logging_obj, seconds=0.25)
+        logging_obj.model_call_details["end_time"] = logging_obj.start_time + datetime.timedelta(seconds=1.0)
+        result: Final = logging_obj._handle_anthropic_messages_response_logging(result=dict(self.MESSAGES_RESPONSE))
+
+        assert logging_obj._response_cost_calculator(result=result) == pytest.approx(self.RATE)
+
+
 class TestGetRouterModelId:
     """Tests for the get_router_model_id helper method."""
 
@@ -3723,9 +3833,9 @@ def test_get_extra_header_tags_returns_none_for_absent_or_empty_header(
     monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["x-org-id"], raising=False)
 
     assert StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": {}}) is None
-    assert StandardLoggingPayloadSetup._get_extra_header_tags(
-        proxy_server_request={"headers": {"x-org-id": ""}}
-    ) is None
+    assert (
+        StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": {"x-org-id": ""}}) is None
+    )
 
 
 def test_get_extra_header_tags_preserves_configured_header_order(
@@ -11939,7 +12049,7 @@ def test_get_additional_headers():
 
 
 def all_fields_present(standard_logging_metadata: StandardLoggingMetadata):
-    for field in StandardLoggingMetadata.__annotations__.keys():
+    for field in StandardLoggingMetadata.__required_keys__:
         assert field in standard_logging_metadata
 
 
@@ -11975,6 +12085,26 @@ def test_get_standard_logging_metadata(metadata_key, metadata_value):
     assert standard_logging_metadata[metadata_key] == metadata_value
 
 
+@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
+def test_module_level_get_standard_logging_metadata_drops_only_a_caller_sent_silent_experiment_flag():
+    from litellm.litellm_core_utils.litellm_logging import get_standard_logging_metadata
+
+    caller_metadata: Final = {
+        "is_silent_experiment": True,
+        "user_api_key_alias": "alias",
+        "usage_object": {"total_tokens": 3},
+        "used_client_oauth_token": True,
+        "kubernetes_pod_routing": {"pod": "a"},
+    }
+
+    result = get_standard_logging_metadata(caller_metadata)
+
+    assert "is_silent_experiment" not in result
+    assert {key: result[key] for key in caller_metadata.keys() - {"is_silent_experiment"}} == {
+        key: value for key, value in caller_metadata.items() if key != "is_silent_experiment"
+    }
+
+
 @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
 def test_get_standard_logging_metadata_user_api_key_hash():
     valid_hash = "a" * 64  # 64 character string
@@ -13801,7 +13931,9 @@ def test_record_native_bedrock_guardrail_bills_each_call_its_own_trace_when_call
         )
         logging_obj.model_call_details["litellm_params"] = {"metadata": shared_metadata}
         logging_obj.record_native_bedrock_guardrail({"trace": trace})
-        recorded = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"]
+        recorded = logging_obj.model_call_details["litellm_params"]["metadata"][
+            "standard_logging_guardrail_information"
+        ]
         costs.append(guardrail_information_cost(recorded))
 
     (first_cost, second_cost) = costs

tests/unit/litellm_core_utils/test_text_completion_conversion.py

@@ -1,10 +1,12 @@
 import asyncio
+from typing import Final
 from unittest.mock import MagicMock, patch
 
 import pytest
 
 import litellm
 from litellm import text_completion
+from litellm.llms.custom_llm import CustomLLM
 from litellm.types.utils import ModelResponse, TextCompletionResponse, Usage
 from litellm.utils import LiteLLMResponseObjectHandler
 
@@ -1298,3 +1300,30 @@ def mock_post(*args, **kwargs):
         "usage": {"prompt_tokens": 2, "total_tokens": 18, "completion_tokens": 16},
     }
     return mock_response
+
+
+def test_text_completion_reads_a_custom_provider_dict_reply(monkeypatch: pytest.MonkeyPatch) -> None:
+    class DictReplyLLM(CustomLLM):
+        def completion(self, *args: object, **kwargs: object) -> dict[str, object]:
+            return {
+                "id": "dict-reply",
+                "object": "chat.completion",
+                "created": 1,
+                "model": "x",
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {"role": "assistant", "content": "hi from a dict reply"},
+                        "finish_reason": "stop",
+                    }
+                ],
+            }
+
+    monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "dictreply", "custom_handler": DictReplyLLM()}])
+    monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "dictreply"])
+    monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "dictreply"])
+
+    response: Final = text_completion(model="dictreply/x", prompt="hi")
+
+    assert response.choices[0].text == "hi from a dict reply"
+    assert response.choices[0].finish_reason == "stop"

tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py

@@ -537,7 +537,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou
         {"type": "content_block_delta", "index": 2, "delta": {"type": "thinking_delta", "thinking": "Step 3."}},
         {"type": "content_block_delta", "index": 2, "delta": {"type": "signature_delta", "signature": "sig-b"}},
         {"type": "content_block_stop", "index": 2},
-        {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 9}},
+        {
+            "type": "message_delta",
+            "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+            "usage": {"output_tokens": 9},
+        },
         {"type": "message_stop"},
     )
 
@@ -558,7 +562,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou
         "thinking_blocks": message["thinking_blocks"],
     }
     replayed: Final = anthropic_messages_pt(
-        messages=[{"role": "user", "content": "Capital of France?"}, next_turn, {"role": "user", "content": "And Spain?"}],
+        messages=[
+            {"role": "user", "content": "Capital of France?"},
+            next_turn,
+            {"role": "user", "content": "And Spain?"},
+        ],
         model="claude-haiku-4-5",
         llm_provider="anthropic",
     )
@@ -3650,3 +3658,79 @@ def test_streamed_end_turn_carries_no_stop_details():
     provider_fields: Final = assembled.choices[0].message.provider_specific_fields or {}
     assert "stop_details" not in provider_fields
     assert "stop_reason" not in provider_fields
+
+
+def _anthropic_chat_request_for_bound_thinking(
+    extra_headers: dict[str, str] | None, drop_params: bool | None = None
+) -> httpx.Request:
+    captured_requests: Final[list[httpx.Request]] = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "msg_bound",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-opus-4-8",
+                "content": [{"type": "text", "text": "Hello"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+
+    thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
+    client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+    try:
+        litellm.completion(
+            model="anthropic/claude-opus-4-8",
+            messages=[{"role": "user", "content": "Hello"}],
+            api_key="test-key",
+            client=client,
+            thinking=thinking,
+            extra_headers=extra_headers,
+            drop_params=drop_params,
+        )
+    finally:
+        client.close()
+    assert thinking == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
+    return captured_requests[0]
+
+
+def _anthropic_chat_body_for_bound_thinking(
+    extra_headers: dict[str, str] | None, drop_params: bool | None = None
+) -> dict[str, object]:
+    return json.loads(_anthropic_chat_request_for_bound_thinking(extra_headers, drop_params).content)
+
+
+def test_anthropic_chat_keeps_block_binding_when_the_final_header_carries_the_binding_beta(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    body: Final = _anthropic_chat_body_for_bound_thinking(
+        {"anthropic-beta": "thinking-binding-controls-2026-08-01"}, drop_params=True
+    )
+    assert body["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
+
+
+@pytest.mark.parametrize("drop_params", [True, False])
+def test_anthropic_chat_adds_the_binding_beta_so_block_binding_is_kept(
+    drop_params: bool, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    monkeypatch.setattr(litellm, "drop_params", False)
+    request: Final = _anthropic_chat_request_for_bound_thinking(None, drop_params=drop_params)
+    assert json.loads(request.content)["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+    }
+    assert "thinking-binding-controls-2026-08-01" in request.headers["anthropic-beta"].split(",")
+
+
+def test_anthropic_chat_sends_block_binding_as_is_without_drop_params(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    monkeypatch.setattr(litellm, "drop_params", False)
+    body: Final = _anthropic_chat_body_for_bound_thinking(None)
+    assert body["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}

tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py

@@ -5,8 +5,10 @@ import base64
 import json
 import logging
 import struct
+import time
 import uuid
-from collections.abc import AsyncIterable, Mapping
+from collections.abc import AsyncIterable, AsyncIterator, Mapping
+from datetime import datetime
 from typing import Final
 from zlib import crc32
 
@@ -446,6 +448,135 @@ async def test_router_aanthropic_messages_streaming_logs_usage_model_and_cost(
     assert payload["completion_tokens"] == max(usage.get("output_tokens", 0) for usage in usages) == 20
 
 
+_PER_SECOND_RATE: Final = 0.5
+_UPSTREAM_PAUSE_SECONDS: Final = 0.3
+
+
+def _priced_router(**pricing: float) -> Router:
+    return Router(
+        model_list=[
+            {
+                "model_name": _ALIAS,
+                "litellm_params": {"model": _ANTHROPIC_MODEL, "api_key": "fake-key", **pricing},
+            }
+        ]
+    )
+
+
+def _per_second_router() -> Router:
+    return _priced_router(input_cost_per_second=_PER_SECOND_RATE, output_cost_per_second=_PER_SECOND_RATE)
+
+
+def _anthropic_sse_event(event: Mapping[str, object]) -> bytes:
+    return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
+
+
+async def _route_like_the_proxy(router: Router, **request: object) -> object:
+    logging_obj, routed_request = litellm.utils.function_setup(
+        original_function="aanthropic_messages",
+        rules_obj=litellm.utils.Rules(),
+        start_time=datetime.now(),
+        litellm_call_id=str(uuid.uuid4()),
+        model=_ALIAS,
+        max_tokens=100,
+        **request,
+    )
+    return await router.aanthropic_messages(**routed_request, litellm_logging_obj=logging_obj)
+
+
+async def _sse_pausing_after_the_first_event() -> AsyncIterator[bytes]:
+    yield _anthropic_sse_event(_STREAM_EVENTS[0])
+    await asyncio.sleep(_UPSTREAM_PAUSE_SECONDS)
+    for event in _STREAM_EVENTS[1:]:
+        yield _anthropic_sse_event(event)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "pricing",
+    [
+        {"input_cost_per_second": _PER_SECOND_RATE, "output_cost_per_second": _PER_SECOND_RATE},
+        {"cost_per_second": _PER_SECOND_RATE},
+    ],
+)
+async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_call_duration(
+    respx_mock, monkeypatch, pricing: Mapping[str, float]
+):
+    messages: Final = _unique_messages()
+    recorder: Final = _RecordingLogger(messages)
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    async def slow_upstream(request: httpx.Request) -> httpx.Response:
+        await asyncio.sleep(_UPSTREAM_PAUSE_SECONDS)
+        return httpx.Response(200, json=_anthropic_body())
+
+    respx_mock.post(url__regex=_ANTHROPIC_URL).mock(side_effect=slow_upstream)
+    started: Final = time.monotonic()
+    await _route_like_the_proxy(_priced_router(**pricing), messages=messages)
+    await _wait_for_payload(recorder)
+    elapsed: Final = time.monotonic() - started
+
+    assert (
+        _PER_SECOND_RATE * _UPSTREAM_PAUSE_SECONDS
+        <= recorder.payloads[0]["response_cost"]
+        <= _PER_SECOND_RATE * elapsed
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_whole_stream(respx_mock, monkeypatch):
+    messages: Final = _unique_messages()
+    recorder: Final = _RecordingLogger(messages)
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+    respx_mock.post(url__regex=_ANTHROPIC_URL).mock(
+        side_effect=lambda request: httpx.Response(
+            200, headers={"content-type": "text/event-stream"}, content=_sse_pausing_after_the_first_event()
+        )
+    )
+    started: Final = time.monotonic()
+    stream: Final = await _route_like_the_proxy(_per_second_router(), messages=messages, stream=True)
+    await _stream_events(stream)
+    await _wait_for_payload(recorder)
+    elapsed: Final = time.monotonic() - started
+
+    assert (
+        _PER_SECOND_RATE * _UPSTREAM_PAUSE_SECONDS
+        <= recorder.payloads[0]["response_cost"]
+        <= _PER_SECOND_RATE * elapsed
+    )
+
+
+async def _wait_for_payloads(recorder: _RecordingLogger, count: int) -> None:
+    async def arrived() -> None:
+        while len(recorder.payloads) < count:
+            await asyncio.sleep(0.05)
+
+    await asyncio.wait_for(arrived(), timeout=30.0)
+
+
+@pytest.mark.asyncio
+async def test_router_aanthropic_messages_bills_a_short_call_on_a_long_context_rate_like_a_deployment_without_one(
+    respx_mock, monkeypatch
+):
+    messages: Final = _unique_messages()
+    recorder: Final = _RecordingLogger(messages)
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+    respx_mock.post(url__regex=_ANTHROPIC_URL).mock(
+        side_effect=lambda request: httpx.Response(200, json=_anthropic_body())
+    )
+
+    await _route_like_the_proxy(_priced_router(), messages=messages)
+    await _route_like_the_proxy(
+        _priced_router(input_cost_per_token_above_200k_tokens=1.0, output_cost_per_token_above_200k_tokens=1.0),
+        messages=messages,
+    )
+    await _wait_for_payloads(recorder, 2)
+
+    costs: Final = {payload["response_cost"] for payload in recorder.payloads}
+    assert len(costs) == 1, costs
+    assert costs.pop() > 0
+
+
 _LARGE_SYSTEM_PROMPT: Final = "This is a comprehensive legal agreement between Party A and Party B. " * 100
 
 

tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Mapping
 from datetime import datetime, timezone
 from pathlib import Path
@@ -6,9 +7,11 @@ from unittest.mock import patch
 
 import httpx
 import pytest
+import respx
 from pydantic import JsonValue
 
 import litellm
+from litellm import anthropic_beta_headers_manager
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.llms.anthropic.messages.native.transformation import (
     AnthropicMessagesConfig,
@@ -498,3 +501,133 @@ def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_re
 
     assert request["messages"][1] == 7
     assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}]
+
+
+BINDING_AND_DISPLAY_BETAS: Final = "thinking-binding-controls-2026-08-01,thinking-display-updates-2026-08-18"
+
+
+def test_messages_headers_add_the_binding_beta_when_thinking_carries_block_binding() -> None:
+    headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta(
+        headers={},
+        optional_params={"thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}},
+        messages=[],
+    )
+    assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",")
+
+
+def test_messages_headers_add_no_binding_beta_without_block_binding() -> None:
+    headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta(
+        headers={}, optional_params={"thinking": {"type": "adaptive"}}, messages=[]
+    )
+    assert "thinking-binding-controls-2026-08-01" not in headers.get("anthropic-beta", "")
+
+
+def _bound_adaptive_request(headers: dict[str, str], drop_params: bool | None = True) -> dict[str, object]:
+    thinking: Final = {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+    optional_params: Final = {"max_tokens": 64, "thinking": thinking}
+    request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
+        model="claude-opus-4-8",
+        messages=[{"role": "user", "content": "hi"}],
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(drop_params=drop_params),
+        headers=headers,
+    )
+    assert thinking == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }, "the caller's thinking dict must survive for a fallback leg"
+    return request
+
+
+def test_messages_request_keeps_beta_gated_thinking_fields_when_the_final_header_carries_the_betas() -> None:
+    request: Final = _bound_adaptive_request({"anthropic-beta": BINDING_AND_DISPLAY_BETAS})
+    assert request["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+
+
+def test_messages_request_drops_beta_gated_thinking_fields_the_final_header_does_not_carry(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    with caplog.at_level(logging.WARNING, logger="LiteLLM"):
+        request: Final = _bound_adaptive_request({"anthropic-beta": "thinking-binding-controls-2026-08-01"})
+    assert request["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
+    assert any("thinking.display=updates" in record.getMessage() for record in caplog.records)
+
+
+def test_messages_request_sends_beta_gated_thinking_fields_as_is_without_drop_params(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "drop_params", False)
+    request: Final = _bound_adaptive_request({}, drop_params=None)
+    assert request["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+
+
+def test_messages_request_drops_clear_at_from_a_mid_conversation_system_turn_without_its_beta() -> None:
+    system_turn: Final = {"role": "system", "content": "be terse", "clear_at": "end_turn"}
+    request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
+        model="claude-opus-4-8",
+        messages=[{"role": "user", "content": "hi"}, system_turn, {"role": "user", "content": "go"}],
+        anthropic_messages_optional_request_params={"max_tokens": 64},
+        litellm_params=GenericLiteLLMParams(drop_params=True),
+        headers={"anthropic-beta": "mid-conversation-system-2026-07-01"},
+    )
+    assert request["messages"][1] == {"role": "system", "content": "be terse"}
+    assert system_turn == {"role": "system", "content": "be terse", "clear_at": "end_turn"}
+
+
+def test_a_provider_without_a_beta_headers_row_forwards_beta_gated_fields_under_drop_params(
+    local_beta_headers_config: None,
+) -> None:
+    thinking: Final = {"type": "enabled", "budget_tokens": 1024, "display": "updates"}
+    request: Final = MinimaxMessagesConfig().transform_anthropic_messages_request(
+        model="MiniMax-M2",
+        messages=[
+            {"role": "user", "content": "hi"},
+            {"role": "assistant", "content": "Hello"},
+            {"role": "user", "content": "how are you"},
+            {"role": "system", "content": "Answer in French", "output_config": {"effort": "low"}},
+        ],
+        anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)},
+        litellm_params=GenericLiteLLMParams(drop_params=True),
+        headers={},
+    )
+    assert request["thinking"] == thinking
+    assert request["messages"][-1]["output_config"] == {"effort": "low"}
+
+
+def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_config(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False)
+    monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
+
+    with respx.mock(assert_all_called=False) as router:
+        remote_config: Final = router.get(litellm.anthropic_beta_headers_url).mock(
+            return_value=httpx.Response(200, json={})
+        )
+        request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
+            model="claude-opus-4-8",
+            messages=[
+                {"role": "user", "content": "hi"},
+                {"role": "system", "content": "be terse", "clear_at": "end_turn"},
+                {"role": "user", "content": "go"},
+            ],
+            anthropic_messages_optional_request_params={"max_tokens": 64},
+            litellm_params=GenericLiteLLMParams(drop_params=True),
+            headers={},
+        )
+
+    assert request["messages"][1] == {"role": "system", "content": "be terse"}
+    assert not remote_config.called

tests/unit/llms/anthropic/messages/native/test_response_cache.py

@@ -2,6 +2,7 @@ import asyncio
 from typing import Any, AsyncIterator, Dict, Final, List
 
 import pytest
+import pytest_asyncio
 
 
 import datetime
@@ -67,7 +68,7 @@ async def _collect(stream: AsyncIterator[bytes]) -> List[bytes]:
     return [chunk async for chunk in stream]
 
 
-@pytest.fixture(autouse=True)
+@pytest_asyncio.fixture(autouse=True, loop_scope="function")
 async def _drain_logging_worker():
     yield
     await GLOBAL_LOGGING_WORKER.flush()

tests/unit/llms/anthropic/test_anthropic_common_utils.py

@@ -4296,7 +4296,14 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
 
 @pytest.mark.usefixtures("local_model_cost_map")
 @pytest.mark.parametrize(
-    ("custom_llm_provider", "model", "requested_thinking", "rejected_keys", "expected_thinking"),
+    (
+        "custom_llm_provider",
+        "model",
+        "requested_thinking",
+        "rejected_keys",
+        "expected_thinking",
+        "provider_rejects_block_binding",
+    ),
     (
         (
             "anthropic",
@@ -4304,6 +4311,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
             {"type": "enabled", "budget_tokens": 2048},
             {"not_a_thinking_key": True},
             {"type": "adaptive"},
+            False,
         ),
         (
             "bedrock",
@@ -4311,6 +4319,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
             {"type": "enabled", "budget_tokens": 2048},
             {"not_a_thinking_key": True},
             {"type": "adaptive"},
+            False,
         ),
         (
             "bedrock",
@@ -4318,6 +4327,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
             {"type": "enabled", "budget_tokens": 2048},
             {"not_a_thinking_key": True},
             {"type": "adaptive"},
+            False,
         ),
         (
             "bedrock",
@@ -4325,6 +4335,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
             {"type": "adaptive", "display": "updates"},
             {"not_a_thinking_key": True, "budget_tokens": 2048},
             {"type": "adaptive", "display": "updates"},
+            False,
         ),
         (
             "vertex_ai",
@@ -4332,6 +4343,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
             {"type": "enabled", "budget_tokens": 2048},
             {"not_a_thinking_key": True},
             {"type": "adaptive"},
+            False,
         ),
         (
             "databricks",
@@ -4339,6 +4351,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding(
             {"type": "enabled", "budget_tokens": 2048},
             {"not_a_thinking_key": True},
             {"type": "adaptive"},
+            True,
         ),
     ),
     ids=(
@@ -4357,6 +4370,7 @@ def test_chat_thinking_drops_unknown_keys_only_under_drop_params(
     requested_thinking: dict[str, object],
     rejected_keys: dict[str, object],
     expected_thinking: dict[str, object],
+    provider_rejects_block_binding: bool,
     drop_params_source: str | None,
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -4371,7 +4385,10 @@ def test_chat_thinking_drops_unknown_keys_only_under_drop_params(
     )
 
     kept_rejected_keys: Final = {} if drop_params_source else rejected_keys
-    assert optional_params["thinking"] == {**expected_thinking, "block_binding": block_binding, **kept_rejected_keys}
+    kept_block_binding: Final = (
+        {} if drop_params_source and provider_rejects_block_binding else {"block_binding": block_binding}
+    )
+    assert optional_params["thinking"] == {**expected_thinking, **kept_block_binding, **kept_rejected_keys}
 
 
 @pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")

tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py

@@ -14,7 +14,7 @@ import litellm
 from litellm.caching.dual_cache import DualCache
 from litellm.caching.llm_caching_handler import LLMClientCache
 from litellm.llms.anthropic.count_tokens import handler as count_handler
-from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION
+from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig
 from litellm.llms.anthropic.prompt_cache_prediction import (
     CountedPromptCachePlan,
     NativePredictionTarget,
@@ -38,7 +38,7 @@ from litellm.proxy import proxy_server
 from litellm.proxy.hooks.prompt_cache_prediction import PromptCacheObserver
 from litellm.proxy.management_endpoints.prompt_cache_prediction import predict_arm
 from litellm.proxy.utils import InternalUsageCache
-from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
+from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, ModelInfo
 from litellm.types.utils import CacheCreationTokenDetails, ModelResponse, PromptTokensDetailsWrapper, Usage
 
 _MODEL: Final = "claude-sonnet-5"
@@ -52,9 +52,12 @@ def _body() -> dict[str, JsonValue]:
         "model": _MODEL,
         "system": "Keep this context",
         "tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
-        "messages": [{"role": "user", "content": [
-            {"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}}
-        ]}],
+        "messages": [
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}}],
+            }
+        ],
     }
 
 
@@ -90,7 +93,8 @@ async def test_observer_records_only_version_supported_by_token_counter(version:
             "httpx_response": httpx.Response(200, request=wire),
             "first_api_call_start_time": datetime.fromtimestamp(1000.0),
             "standard_logging_object": {
-                "status": "success", "model_id": _DEPLOYMENT,
+                "status": "success",
+                "model_id": _DEPLOYMENT,
                 "metadata": {"user_api_key_hash": _CALLER},
             },
         },
@@ -106,13 +110,16 @@ async def test_observer_records_only_version_supported_by_token_counter(version:
         assert await lookup(cache, other_scope, prefix, now=1010.0) is None
 
 
-@pytest.mark.parametrize("headers, supported", [
-    ({}, True),
-    ({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True),
-    ({"anthropic-version": "2099-01-01"}, False),
-    ({"Anthropic-Beta": ""}, False),
-    ({"anthropic-beta": "future-feature"}, False),
-])
+@pytest.mark.parametrize(
+    "headers, supported",
+    [
+        ({}, True),
+        ({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True),
+        ({"anthropic-version": "2099-01-01"}, False),
+        ({"Anthropic-Beta": ""}, False),
+        ({"anthropic-beta": "future-feature"}, False),
+    ],
+)
 def test_prediction_header_eligibility(headers: Mapping[str, str], supported: bool) -> None:
     assert supported_prediction_headers(headers) is supported
 
@@ -152,49 +159,89 @@ async def test_provider_count_uses_same_version_and_preserves_native_input(
 @pytest.mark.parametrize("source", ["static", "database"])
 @pytest.mark.asyncio
 async def test_environment_credential_matches_native_count_and_observed_scope(
-    source: str, monkeypatch: pytest.MonkeyPatch,
+    source: str,
+    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     monkeypatch.setenv("LIT7658_PROVIDER_KEY", _KEY)
     monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
     monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
     params: Final = {
-        "model": f"anthropic/{_MODEL}", "api_key": "os.environ/LIT7658_PROVIDER_KEY",
+        "model": f"anthropic/{_MODEL}",
+        "api_key": "os.environ/LIT7658_PROVIDER_KEY",
         "api_base": "https://api.anthropic.com",
     }
-    router: Final = litellm.Router(model_list=[{
-        "model_name": "test-native", "litellm_params": dict(params), "model_info": {"id": _DEPLOYMENT},
-    }] if source == "static" else [], num_retries=0)
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "test-native",
+                "litellm_params": dict(params),
+                "model_info": {"id": _DEPLOYMENT},
+            }
+        ]
+        if source == "static"
+        else [],
+        num_retries=0,
+    )
     if source == "database":
         monkeypatch.setattr(proxy_server, "llm_router", router)
-        assert proxy_server.ProxyConfig()._add_deployment([SimpleNamespace(
-            model_id=_DEPLOYMENT, model_name="test-native", model_info={}, litellm_params=dict(params),
-        )]) == 1
+        assert (
+            proxy_server.ProxyConfig()._add_deployment(
+                [
+                    SimpleNamespace(
+                        model_id=_DEPLOYMENT,
+                        model_name="test-native",
+                        model_info={},
+                        litellm_params=dict(params),
+                    )
+                ]
+            )
+            == 1
+        )
     deployment: Final = router.get_deployment(_DEPLOYMENT)
     assert deployment is not None
     target: Final = resolve_prediction_target(deployment.litellm_params)
     assert isinstance(target, NativePredictionTarget)
     body: Final = _body()
     with respx.mock() as upstream:
-        native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond(200, json={
-            "id": "msg_test", "type": "message", "role": "assistant", "model": _MODEL,
-            "content": [{"type": "text", "text": "Hello"}], "stop_reason": "end_turn", "stop_sequence": None,
-            "usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300},
-        })
+        native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond(
+            200,
+            json={
+                "id": "msg_test",
+                "type": "message",
+                "role": "assistant",
+                "model": _MODEL,
+                "content": [{"type": "text", "text": "Hello"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300},
+            },
+        )
         counter: Final = upstream.post("https://api.anthropic.com/v1/messages/count_tokens").respond(
-            200, json={"input_tokens": 311},
+            200,
+            json={"input_tokens": 311},
         )
         await router.aanthropic_messages(
-            model="test-native", max_tokens=1, **{key: value for key, value in body.items() if key != "model"},
+            model="test-native",
+            max_tokens=1,
+            **{key: value for key, value in body.items() if key != "model"},
         )
         assert await count_prompt_tokens(target.model, target.api_key, body) == 311
     assert native.call_count == counter.call_count == 1
     assert native.calls.last.request.headers["x-api-key"] == counter.calls.last.request.headers["x-api-key"] == _KEY
-    observed: Final = parse_observed_cache(native.calls.last.request, ModelResponse(
-        model=_MODEL, usage=Usage(
-            prompt_tokens=311, completion_tokens=1, total_tokens=312,
-            prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300),
+    observed: Final = parse_observed_cache(
+        native.calls.last.request,
+        ModelResponse(
+            model=_MODEL,
+            usage=Usage(
+                prompt_tokens=311,
+                completion_tokens=1,
+                total_tokens=312,
+                prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300),
+            ),
         ),
-    ), _CALLER, _DEPLOYMENT)
+        _CALLER,
+        _DEPLOYMENT,
+    )
     assert observed is not None
     assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model)
 
@@ -202,15 +249,26 @@ async def test_environment_credential_matches_native_count_and_observed_scope(
 @pytest.mark.parametrize("inline_key", [None, _KEY])
 @pytest.mark.asyncio
 async def test_named_credential_is_explicitly_unsupported_before_count(
-    inline_key: str | None, monkeypatch: pytest.MonkeyPatch,
+    inline_key: str | None,
+    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    monkeypatch.setattr(litellm, "credential_list", [CredentialItem(
-        credential_name="test-named", credential_info={}, credential_values={"api_key": "test-named-provider-key"},
-    )])
+    monkeypatch.setattr(
+        litellm,
+        "credential_list",
+        [
+            CredentialItem(
+                credential_name="test-named",
+                credential_info={},
+                credential_values={"api_key": "test-named-provider-key"},
+            )
+        ],
+    )
     deployment: Final = Deployment(
         model_name="test-native",
         litellm_params=LiteLLM_Params(
-            model=f"anthropic/{_MODEL}", api_key=inline_key, litellm_credential_name="test-named",
+            model=f"anthropic/{_MODEL}",
+            api_key=inline_key,
+            litellm_credential_name="test-named",
         ),
         model_info=ModelInfo(id=_DEPLOYMENT),
     )
@@ -234,8 +292,7 @@ def _cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan:
 
 
 def _text(text: str, ttl: str | None = None) -> dict[str, JsonValue]:
-    return {"type": "text", "text": text,
-            **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})}
+    return {"type": "text", "text": text, **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})}
 
 
 def _prompt(*blocks: dict[str, JsonValue], role: str = "user", **options: JsonValue) -> dict[str, JsonValue]:
@@ -253,16 +310,19 @@ def test_public_predictor_preserves_string_message_policy(text: str, supported:
 
 def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None:
     body: Final = _prompt(
-        _text("First turn", "5m"), system=[_text("Stable instructions", "1h")],
-        tools=[{"name": "lookup", "input_schema": {"type": "object"},
-                "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
+        _text("First turn", "5m"),
+        system=[_text("Stable instructions", "1h")],
+        tools=[
+            {"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral", "ttl": "1h"}}
+        ],
     )
     plan: Final = _cache_plan(body)
     changed: Final = _cache_plan({**body, "system": [_text("Changed instructions", "1h")]})
     assert tuple(marker.ttl_seconds for marker in plan.breakpoints) == (3600, 3600, 300)
     assert plan.breakpoints[0].fingerprint == changed.breakpoints[0].fingerprint
-    assert all(left.fingerprint != right.fingerprint for left, right
-               in zip(plan.breakpoints[1:], changed.breakpoints[1:]))
+    assert all(
+        left.fingerprint != right.fingerprint for left, right in zip(plan.breakpoints[1:], changed.breakpoints[1:])
+    )
     assert plan.breakpoints[0].prefix_body == {
         "tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
         "messages": [],
@@ -270,51 +330,80 @@ def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None:
     assert parse_prompt(body) is None
 
 
-@pytest.mark.parametrize("kind, added, matches", [
-    ("text", 19, True), ("text", 20, False), ("tool_use", 30, True),
-    ("tool_result", 30, True),
-])
+@pytest.mark.parametrize(
+    "kind, added, matches",
+    [
+        ("text", 19, True),
+        ("text", 20, False),
+        ("tool_use", 30, True),
+        ("tool_result", 30, True),
+    ],
+)
 def test_cache_plan_lookback_counts_native_positions(
-    kind: str, added: int, matches: bool,
+    kind: str,
+    added: int,
+    matches: bool,
 ) -> None:
     previous: Final = _cache_plan(_body())
     appended: Final[list[dict[str, JsonValue]]] = [
         {"type": "tool_use", "id": f"tool_{index}", "name": "lookup", "input": {}}
-        if kind == "tool_use" else
-        {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"}
-        if kind == "tool_result" else
-        {"type": "text", "text": f"Added {index}"}
+        if kind == "tool_use"
+        else {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"}
+        if kind == "tool_result"
+        else {"type": "text", "text": f"Added {index}"}
         for index in range(added)
     ]
-    current: Final = _cache_plan({**_body(), **_prompt(
-        _text("A cacheable prefix"), *appended[:-1],
-        {**appended[-1], "cache_control": {"type": "ephemeral"}},
-    )})
-    assert (previous.breakpoints[0].fingerprint
-            in current.breakpoints[0].lookback_fingerprints) is matches
+    current: Final = _cache_plan(
+        {
+            **_body(),
+            **_prompt(
+                _text("A cacheable prefix"),
+                *appended[:-1],
+                {**appended[-1], "cache_control": {"type": "ephemeral"}},
+            ),
+        }
+    )
+    assert (previous.breakpoints[0].fingerprint in current.breakpoints[0].lookback_fingerprints) is matches
 
 
-@pytest.mark.parametrize("change, same_prefix, same_content", [
-    ("tool_order", False, False), ("effort", False, False),
-    ("standard_speed", True, True), ("ttl", False, True),
-])
+@pytest.mark.parametrize(
+    "change, same_prefix, same_content",
+    [
+        ("tool_order", False, False),
+        ("effort", False, False),
+        ("standard_speed", True, True),
+        ("ttl", False, True),
+    ],
+)
 def test_cache_plan_identity_respects_settings_and_preserves_content(
-    change: str, same_prefix: bool, same_content: bool,
+    change: str,
+    same_prefix: bool,
+    same_content: bool,
 ) -> None:
     tool_input: Final[dict[str, JsonValue]] = {"a": 1, "b": 2, "cache_control": {"ttl": "user-data"}}
     block: Final[dict[str, JsonValue]] = {
-        "type": "tool_use", "id": "tool_1", "name": "lookup", "input": tool_input,
+        "type": "tool_use",
+        "id": "tool_1",
+        "name": "lookup",
+        "input": tool_input,
         "cache_control": {"type": "ephemeral", "ttl": "5m"},
     }
     changed_block: Final = (
-        {**block, "input": dict(reversed(tool_input.items()))} if change == "tool_order" else
-        {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} if change == "ttl" else block
+        {**block, "input": dict(reversed(tool_input.items()))}
+        if change == "tool_order"
+        else {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}}
+        if change == "ttl"
+        else block
     )
     before: Final = _cache_plan(_prompt(block, role="assistant", output_config={"effort": "low"})).breakpoints[0]
-    after: Final = _cache_plan(_prompt(
-        changed_block, role="assistant", output_config={"effort": "high" if change == "effort" else "low"},
-        **({"speed": "standard"} if change == "standard_speed" else {}),
-    )).breakpoints[0]
+    after: Final = _cache_plan(
+        _prompt(
+            changed_block,
+            role="assistant",
+            output_config={"effort": "high" if change == "effort" else "low"},
+            **({"speed": "standard"} if change == "standard_speed" else {}),
+        )
+    ).breakpoints[0]
     assert (before.fingerprint == after.fingerprint) is same_prefix
     assert (before.fingerprint in after.lookback_fingerprints) is same_prefix
     assert (before.content_fingerprint == after.content_fingerprint) is same_content
@@ -325,30 +414,41 @@ def test_cache_plan_identity_respects_settings_and_preserves_content(
 
 def test_cache_plan_automatic_cache_and_thinking_use_last_cacheable_block() -> None:
     body: Final = _prompt(
-        _text("A stable answer"), {"type": "thinking", "thinking": "Thinking", "signature": "signature"},
-        role="assistant", thinking={"type": "adaptive"}, cache_control={"type": "ephemeral", "ttl": "1h"},
+        _text("A stable answer"),
+        {"type": "thinking", "thinking": "Thinking", "signature": "signature"},
+        role="assistant",
+        thinking={"type": "adaptive"},
+        cache_control={"type": "ephemeral", "ttl": "1h"},
     )
     plan: Final = _cache_plan(body)
     assert len(plan.breakpoints) == 1
     assert plan.breakpoints[0].ttl_seconds == 3600
     assert plan.breakpoints[0].prefix_body == {
         "thinking": {"type": "adaptive"},
-        "messages": [{"role": "assistant", "content": [
-            {"type": "text", "text": "A stable answer"},
-        ]}],
+        "messages": [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "A stable answer"},
+                ],
+            }
+        ],
     }
     assert parse_prompt(body) is None
 
 
-@pytest.mark.parametrize("body, reason", [
-    (_prompt({"type": "image"}), "unsupported_prompt_shape"),
-    ({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"),
-    ({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}},
-     "conflicting_cache_ttl"),
-    (_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"),
-])
+@pytest.mark.parametrize(
+    "body, reason",
+    [
+        (_prompt({"type": "image"}), "unsupported_prompt_shape"),
+        ({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"),
+        ({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}}, "conflicting_cache_ttl"),
+        (_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"),
+    ],
+)
 def test_cache_plan_unsupported_is_explicit(
-    body: Mapping[str, JsonValue], reason: str,
+    body: Mapping[str, JsonValue],
+    reason: str,
 ) -> None:
     result: Final = parse_cache_plan(body)
     assert isinstance(result, UnsupportedCachePlan)
@@ -356,30 +456,46 @@ def test_cache_plan_unsupported_is_explicit(
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("model, counts, reason", [
-    (None, (100, 150, 200), None),
-    (None, (100, 201, 200), "inconsistent_prefix_token_count"),
-    (None, (151, 150, 200), "inconsistent_prefix_token_count"),
-    (None, (None, 150, 200), "token_count_unavailable"),
-    ("claude-opus-5", (100, 150, 200), None),
-    ("claude-sonnet-5", (100, 150, 200), None),
-    ("declared-cache-model", (100, 150, 200), None),
-    ("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"),
-    ("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
-    ("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
-])
+@pytest.mark.parametrize(
+    "model, counts, reason",
+    [
+        (None, (100, 150, 200), None),
+        (None, (100, 201, 200), "inconsistent_prefix_token_count"),
+        (None, (151, 150, 200), "inconsistent_prefix_token_count"),
+        (None, (None, 150, 200), "token_count_unavailable"),
+        ("claude-opus-5", (100, 150, 200), None),
+        ("claude-sonnet-5", (100, 150, 200), None),
+        ("declared-cache-model", (100, 150, 200), None),
+        ("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"),
+        ("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
+        ("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
+    ],
+)
 async def test_cache_plan_count_conserves_total_and_rejects_unknown(
-    model: str | None, counts: tuple[int | None, int | None, int], reason: str | None,
+    model: str | None,
+    counts: tuple[int | None, int | None, int],
+    reason: str | None,
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    monkeypatch.setitem(litellm.model_cost, "declared-cache-model", {
-        "litellm_provider": "anthropic", "mode": "chat", "supports_thinking_cache_preservation": True,
-    })
-    plan: Final = _cache_plan(_prompt(
-        {"type": "thinking", "thinking": "Retained thought", "signature": "signature"}
-        if model else _text("first", "5m"),
-        _text("second", "5m"), _text("uncached"), role="assistant" if model else "user",
-    ))
+    monkeypatch.setitem(
+        litellm.model_cost,
+        "declared-cache-model",
+        {
+            "litellm_provider": "anthropic",
+            "mode": "chat",
+            "supports_thinking_cache_preservation": True,
+        },
+    )
+    plan: Final = _cache_plan(
+        _prompt(
+            {"type": "thinking", "thinking": "Retained thought", "signature": "signature"}
+            if model
+            else _text("first", "5m"),
+            _text("second", "5m"),
+            _text("uncached"),
+            role="assistant" if model else "user",
+        )
+    )
 
     async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
         assert reason != "unsupported_thinking_cache_semantics", "Unverified thinking retention must skip counting"
@@ -401,12 +517,15 @@ async def test_cache_plan_count_conserves_total_and_rejects_unknown(
 @pytest.mark.parametrize("section", ["system", "tools"])
 @pytest.mark.parametrize("rejects_prefix", (False, True))
 async def test_native_count_preserves_settings_and_requires_every_prefix(
-    section: str, rejects_prefix: bool, monkeypatch: pytest.MonkeyPatch,
+    section: str,
+    rejects_prefix: bool,
+    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
     monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
     params: Final = LiteLLM_Params(
-        model=f"anthropic/{_MODEL}", api_key=_KEY,
+        model=f"anthropic/{_MODEL}",
+        api_key=_KEY,
         api_base="https://gateway.example/v1/messages",
     )
     target: Final = resolve_baseline_prediction_target(params)
@@ -415,19 +534,38 @@ async def test_native_count_preserves_settings_and_requires_every_prefix(
     assert not isinstance(resolve_prediction_target(params), NativePredictionTarget)
     body: Final = _body()
     marker: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"}
-    body[section] = ([_text("A cached system", "1h")] if section == "system" else [{
-        "name": "lookup", "input_schema": {"type": "object"}, "cache_control": marker,
-    }])
-    plan: Final = _cache_plan({**body, **_prompt(
-        _text("A later prefix", "5m"), _text("An uncached suffix"),
-        thinking={"type": "adaptive"}, tool_choice={"type": "auto"}, output_config={"effort": "high"},
-    )})
+    body[section] = (
+        [_text("A cached system", "1h")]
+        if section == "system"
+        else [
+            {
+                "name": "lookup",
+                "input_schema": {"type": "object"},
+                "cache_control": marker,
+            }
+        ]
+    )
+    plan: Final = _cache_plan(
+        {
+            **body,
+            **_prompt(
+                _text("A later prefix", "5m"),
+                _text("An uncached suffix"),
+                thinking={"type": "adaptive"},
+                tool_choice={"type": "auto"},
+                output_config={"effort": "high"},
+            ),
+        }
+    )
     assert len(plan.breakpoints) == 2
     assert plan.breakpoints[0].prefix_body["messages"] == []
 
     async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
         return await count_prompt_tokens(
-            model, api_key, {**body, "max_tokens": 100}, api_base=target.api_base,
+            model,
+            api_key,
+            {**body, "max_tokens": 100},
+            api_base=target.api_base,
         )
 
     with respx.mock(assert_all_called=False) as upstream:
@@ -436,12 +574,16 @@ async def test_native_count_preserves_settings_and_requires_every_prefix(
             upstream.post(endpoint, json={**body, "model": _MODEL}).respond(
                 400 if rejects_prefix and index == 1 else 200,
                 json={"detail": {"error": "messages parameter is required"}}
-                if rejects_prefix and index == 1 else {"input_tokens": tokens},
+                if rejects_prefix and index == 1
+                else {"input_tokens": tokens},
+            )
+            for index, (body, tokens) in enumerate(
+                (
+                    (plan.full_body, 6000),
+                    (plan.breakpoints[0].prefix_body, 5000),
+                    (plan.breakpoints[1].prefix_body, 5800),
+                )
             )
-            for index, (body, tokens) in enumerate((
-                (plan.full_body, 6000), (plan.breakpoints[0].prefix_body, 5000),
-                (plan.breakpoints[1].prefix_body, 5800),
-            ))
         )
         unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1})
         result: Final = await count_cache_plan(target.model, target.api_key, plan, count)
@@ -474,3 +616,32 @@ def test_native_baseline_body_drops_a_non_numeric_budget_without_logging_a_drop(
     assert body is not None
     assert body["thinking"] == {"type": "adaptive"}
     assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == []
+
+
+def test_native_baseline_body_matches_the_sent_body_when_drop_params_meets_an_auto_added_beta(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "drop_params", True)
+    thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
+    messages: Final = [{"role": "user", "content": "Say hi in French"}]
+    predicted: Final = prepare_native_baseline_body(
+        {"model": "claude-opus-4-8", "messages": messages, "max_tokens": 64, "thinking": thinking}, "claude-opus-4-8"
+    )
+    config: Final = AnthropicMessagesConfig()
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={},
+        model="claude-opus-4-8",
+        messages=[dict(message) for message in messages],
+        optional_params={"max_tokens": 64, "thinking": dict(thinking)},
+        litellm_params={},
+        api_key="sk-ant-test",
+    )
+    sent: Final = config.transform_anthropic_messages_request(
+        model="claude-opus-4-8",
+        messages=[dict(message) for message in messages],
+        anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)},
+        litellm_params=GenericLiteLLMParams(),
+        headers=headers,
+    )
+    assert predicted is not None
+    assert predicted["thinking"] == sent["thinking"] == thinking

tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py

@@ -6,6 +6,7 @@ sys.path.insert(
 )
 
 import json
+from typing import Final
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -272,3 +273,71 @@ def test_completion_surfaces_the_status_and_headers_of_a_rejected_request():
     assert rejected.value.status_code == 400
     assert rejected.value.litellm_response_headers["x-request-id"] == "req-1"
     assert "max_tokens too large" in str(rejected.value)
+
+
+@respx.mock
+@pytest.mark.parametrize("drop_params", [True, False])
+def test_completion_adds_the_binding_beta_so_block_binding_is_kept(
+    drop_params: bool, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "drop_params", False)
+    route: Final = respx.post("https://example-resource.services.ai.azure.com/anthropic/v1/messages").mock(
+        return_value=Response(
+            200,
+            json={
+                "id": "msg_bound",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-4-6",
+                "content": [{"type": "text", "text": "Hello"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+    )
+    block_binding: Final = {"prefix_mismatch_behavior": "error"}
+
+    litellm.completion(
+        model="azure_ai/claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "hi"}],
+        api_base="https://example-resource.services.ai.azure.com/anthropic",
+        api_key="azure-test-key",
+        thinking={"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding},
+        drop_params=drop_params,
+    )
+
+    sent: Final = json.loads(route.calls.last.request.content)
+    assert sent["thinking"] == {"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding}
+    assert "thinking-binding-controls-2026-08-01" in route.calls.last.request.headers["anthropic-beta"].split(",")
+
+
+@respx.mock
+def test_completion_sends_the_binding_beta_next_to_the_callers_own_betas(local_beta_headers_config: None) -> None:
+    route: Final = respx.post("https://example-resource.services.ai.azure.com/anthropic/v1/messages").mock(
+        return_value=Response(
+            200,
+            json={
+                "id": "msg_bound",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-4-6",
+                "content": [{"type": "text", "text": "Hello"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+    )
+
+    litellm.completion(
+        model="azure_ai/claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "hi"}],
+        api_base="https://example-resource.services.ai.azure.com/anthropic",
+        api_key="azure-test-key",
+        thinking={"type": "enabled", "budget_tokens": 1024, "block_binding": {"prefix_mismatch_behavior": "error"}},
+        extra_headers={"anthropic-beta": "context-1m-2025-08-07"},
+    )
+
+    sent_betas: Final = set(route.calls.last.request.headers["anthropic-beta"].split(","))
+    assert {"context-1m-2025-08-07", "thinking-binding-controls-2026-08-01"} <= sent_betas

tests/unit/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py

@@ -1,5 +1,7 @@
 import json
+from collections.abc import Mapping
 from datetime import datetime
+from typing import Final
 from unittest.mock import MagicMock
 
 import httpx
@@ -519,6 +521,162 @@ def test_cohere_rerank_relay_is_costed_per_search_unit():
     assert logging_obj.response_cost_calculator(result=result) == pytest.approx(2 * per_query)
 
 
+COHERE_EMBED_BODY: Final[Mapping[str, object]] = {
+    "id": "embed-1",
+    "texts": ["hello world"],
+    "embeddings": {"float": [[0.1, 0.2]]},
+    "meta": {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 2, "image_tokens": 0}},
+    "response_type": "embeddings_by_type",
+}
+COHERE_V1_EMBED_META: Final[Mapping[str, object]] = {
+    "api_version": {"version": "1"},
+    "billed_units": {"input_tokens": 2, "image_tokens": 0},
+}
+TWO_VECTORS: Final = [[0.1, 0.2], [0.3, 0.4]]
+
+
+def _cohere_embed_relay_result(
+    body: Mapping[str, object], native_path: str = "providers/cohere/v2/embed"
+) -> tuple[object, Logging]:
+    return _relay_logging_result(AzureAIPassthroughConfig(), "embed-v-4-0", native_path, body)
+
+
+def _foundry_embed_price(price_key: str) -> float:
+    return litellm.get_model_info("azure_ai/embed-v-4-0")[price_key]
+
+
+def test_native_cohere_embed_relay_is_costed_per_billed_input_token() -> None:
+    result, logging_obj = _cohere_embed_relay_result(COHERE_EMBED_BODY)
+    per_token: Final = _foundry_embed_price("input_cost_per_token")
+
+    assert isinstance(result, EmbeddingResponse)
+    assert logging_obj.call_type == "aembedding"
+    assert per_token > 0
+    assert logging_obj.response_cost_calculator(result=result) == pytest.approx(2 * per_token)
+
+
+@pytest.mark.parametrize(
+    "body",
+    [
+        {
+            "id": "embed-1",
+            "texts": ["hello world", "second text"],
+            "embeddings": {"float": TWO_VECTORS},
+            "meta": COHERE_V1_EMBED_META,
+            "response_type": "embeddings_by_type",
+        },
+        {
+            "id": "embed-1",
+            "texts": ["hello world", "second text"],
+            "embeddings": TWO_VECTORS,
+            "meta": COHERE_V1_EMBED_META,
+            "response_type": "embeddings_floats",
+        },
+    ],
+    ids=["embeddings_by_type", "embeddings_floats"],
+)
+def test_native_cohere_v1_embed_relay_is_costed_per_billed_input_token(body: Mapping[str, object]) -> None:
+    result, logging_obj = _cohere_embed_relay_result(body, "providers/cohere/v1/embed")
+    per_token: Final = _foundry_embed_price("input_cost_per_token")
+
+    assert isinstance(result, EmbeddingResponse)
+    assert [(row["index"], row["embedding"]) for row in result.data] == list(enumerate(TWO_VECTORS))
+    assert logging_obj.call_type == "aembedding"
+    assert per_token > 0
+    assert logging_obj.response_cost_calculator(result=result) == pytest.approx(2 * per_token)
+
+
+def test_native_cohere_embed_relay_prices_an_image_at_the_input_tokens_foundry_already_folded_it_into() -> None:
+    body: Final = {
+        **COHERE_EMBED_BODY,
+        "texts": [],
+        "images": [{"width": 64, "height": 64, "format": "image/png", "bit_depth": 8}],
+        "meta": {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 24, "image_tokens": 6}},
+    }
+    result, logging_obj = _cohere_embed_relay_result(body)
+    per_token: Final = _foundry_embed_price("input_cost_per_token")
+    per_image_token: Final = _foundry_embed_price("input_cost_per_image_token")
+
+    assert isinstance(result, EmbeddingResponse)
+    assert per_token > 0
+    assert per_image_token > 0
+    assert logging_obj.response_cost_calculator(result=result) == pytest.approx(24 * per_token)
+
+
+
+def test_native_cohere_embed_relay_rows_keep_their_embedding_type() -> None:
+    result, _ = _cohere_embed_relay_result({**COHERE_EMBED_BODY, "embeddings": {"float": TWO_VECTORS, "int8": [[1, 2]]}})
+
+    assert isinstance(result, EmbeddingResponse)
+    assert [(row["type"], row["index"], row["embedding"]) for row in result.data] == [
+        ("float", 0, TWO_VECTORS[0]),
+        ("float", 1, TWO_VECTORS[1]),
+        ("int8", 0, [1, 2]),
+    ]
+
+@pytest.mark.parametrize("texts", [None, "hello world", [1, 2], ["hello", 3]])
+def test_native_cohere_embed_relay_with_billed_units_is_costed_whatever_texts_holds(texts: object) -> None:
+    result, logging_obj = _cohere_embed_relay_result({**COHERE_EMBED_BODY, "texts": texts})
+
+    assert isinstance(result, EmbeddingResponse)
+    assert logging_obj.call_type == "aembedding"
+    assert logging_obj.response_cost_calculator(result=result) == pytest.approx(
+        2 * _foundry_embed_price("input_cost_per_token")
+    )
+
+
+def test_native_cohere_embed_relay_without_texts_is_costed_per_billed_input_token() -> None:
+    body: Final = {key: value for key, value in COHERE_EMBED_BODY.items() if key != "texts"}
+    result, logging_obj = _cohere_embed_relay_result(body)
+
+    assert isinstance(result, EmbeddingResponse)
+    assert logging_obj.call_type == "aembedding"
+    assert logging_obj.response_cost_calculator(result=result) == pytest.approx(
+        2 * _foundry_embed_price("input_cost_per_token")
+    )
+
+
+@pytest.mark.parametrize(
+    ("texts", "counted_texts"),
+    [
+        (["hello world", "a longer second text"], ["hello world", "a longer second text"]),
+        (["before <|endoftext|> after"], ["before <|endoftext|> after"]),
+        ("hello world", []),
+        (["hello", 3], []),
+        (None, []),
+    ],
+)
+def test_native_cohere_embed_relay_without_billed_units_is_costed_from_the_echoed_texts(
+    texts: object, counted_texts: list[str]
+) -> None:
+    result, logging_obj = _cohere_embed_relay_result({**COHERE_EMBED_BODY, "texts": texts, "meta": {}})
+    per_token: Final = _foundry_embed_price("input_cost_per_token")
+    echoed_tokens: Final = sum(len(litellm.encoding.encode(text, disallowed_special=())) for text in counted_texts)
+
+    assert isinstance(result, EmbeddingResponse)
+    assert logging_obj.call_type == "aembedding"
+    assert result.usage.prompt_tokens == echoed_tokens
+    assert logging_obj.response_cost_calculator(result=result) == pytest.approx(echoed_tokens * per_token)
+
+
+@pytest.mark.parametrize(
+    "body",
+    [
+        {"message": "no embeddings here"},
+        {**COHERE_EMBED_BODY, "embeddings": None},
+        {**COHERE_EMBED_BODY, "embeddings": "vectors"},
+        {**COHERE_EMBED_BODY, "meta": "not an object"},
+    ],
+)
+def test_native_cohere_embed_relay_with_a_non_embed_body_keeps_the_passthrough_object_and_call_type(
+    body: Mapping[str, object],
+) -> None:
+    result, logging_obj = _cohere_embed_relay_result(body)
+
+    assert result == {"response": body}
+    assert logging_obj.call_type == "allm_passthrough_route"
+
+
 def test_image_generation_relay_is_costed_per_image():
     result, logging_obj = _relay_logging_result(
         AzureAIPassthroughConfig(), "FLUX.2-pro", "openai/deployments/FLUX.2-pro/images/generations", IMAGE_BODY

tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py

@@ -2008,3 +2008,25 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co
     assert not AmazonAnthropicClaudeConfig().uses_opt_in_native_output(
         model, optional_params, {"model_info": model_info}
     )
+
+
+@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
+@pytest.mark.parametrize("drop_params", [True, False])
+def test_bedrock_chat_invoke_adds_the_binding_beta_so_block_binding_is_kept(
+    drop_params: bool, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "drop_params", False)
+    block_binding: Final = {"prefix_mismatch_behavior": "error"}
+    result: Final = AmazonAnthropicClaudeConfig().transform_request(
+        model="us.anthropic.claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "Hello"}],
+        optional_params={
+            "max_tokens": 2048,
+            "thinking": {"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding},
+        },
+        litellm_params={"drop_params": drop_params},
+        headers={},
+    )
+
+    assert result["thinking"] == {"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding}
+    assert "thinking-binding-controls-2026-08-01" in result["anthropic_beta"]

tests/unit/llms/bedrock/chat/test_converse_transformation.py

@@ -25,9 +25,9 @@ from litellm.llms.anthropic.chat.transformation import (
     NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS,
     NATIVE_OUTPUT_MAX_UNION_PARAMETERS,
 )
-from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
+from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject
 from litellm.types.llms.openai import AllMessageValues
@@ -5356,7 +5356,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key(
     assert ("outputConfig" in request) is forwarded
 
 
-
 def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch):
     monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
     monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@@ -10673,7 +10672,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere
 
 
 _CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = {
-    "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}}
+    "textFormat": {
+        "type": "json_schema",
+        "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}},
+    }
 }
 _BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string"
 
@@ -10843,3 +10845,121 @@ def test_bedrock_deepseek_r1_drop_of_an_unadvertised_param_does_not_warn(
         )
 
     assert _warnings_naming(caplog, param) == ()
+
+
+def _converse_bound_thinking_request(
+    headers: dict[str, str] | None, body_betas: tuple[str, ...] = (), drop_params: bool = True
+) -> dict[str, object]:
+    thinking: Final = {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+    data: Final = AmazonConverseConfig()._transform_request_helper(
+        model="anthropic.claude-opus-4-8-v1:0",
+        system_content_blocks=[],
+        optional_params={"thinking": thinking, **({"anthropic_beta": list(body_betas)} if body_betas else {})},
+        messages=[{"role": "user", "content": "hi"}],
+        headers=headers,
+        drop_params=drop_params,
+    )
+    assert thinking == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }, "the caller's thinking dict must survive for a fallback leg"
+    return data["additionalModelRequestFields"]
+
+
+def test_converse_keeps_beta_gated_thinking_fields_the_bedrock_converse_row_carries(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    fields: Final = _converse_bound_thinking_request(
+        {"anthropic-beta": "thinking-binding-controls-2026-08-01,thinking-display-updates-2026-08-18"}
+    )
+    assert fields["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+
+
+def test_converse_adds_the_thinking_betas_so_gated_thinking_fields_are_kept(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    fields: Final = _converse_bound_thinking_request(None)
+    assert fields["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+    assert sorted(fields["anthropic_beta"]) == [
+        "thinking-binding-controls-2026-08-01",
+        "thinking-display-updates-2026-08-18",
+    ]
+
+
+def test_converse_keeps_the_callers_body_betas_next_to_the_added_thinking_betas(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    fields: Final = _converse_bound_thinking_request(None, body_betas=("context-1m-2025-08-07",))
+    assert sorted(fields["anthropic_beta"]) == [
+        "context-1m-2025-08-07",
+        "thinking-binding-controls-2026-08-01",
+        "thinking-display-updates-2026-08-18",
+    ]
+
+
+def test_converse_sends_beta_gated_thinking_fields_as_is_without_drop_params(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    monkeypatch.setattr(litellm, "drop_params", False)
+    fields: Final = _converse_bound_thinking_request(None, drop_params=False)
+    assert fields["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+
+
+def test_converse_keeps_beta_gated_thinking_fields_the_body_anthropic_beta_carries(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    fields: Final = _converse_bound_thinking_request(
+        None, body_betas=("thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18")
+    )
+    assert fields["thinking"] == {
+        "type": "adaptive",
+        "block_binding": {"prefix_mismatch_behavior": "error"},
+        "display": "updates",
+    }
+
+
+def test_converse_drops_body_betas_the_bedrock_converse_row_rejects_when_adding_thinking_betas(
+    local_beta_headers_config: None,
+) -> None:
+    fields: Final = _converse_bound_thinking_request(
+        None, body_betas=("per-turn-control-2026-07-01", "context-1m-2025-08-07")
+    )
+    assert sorted(fields["anthropic_beta"]) == [
+        "context-1m-2025-08-07",
+        "thinking-binding-controls-2026-08-01",
+        "thinking-display-updates-2026-08-18",
+    ]
+
+
+def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_betas(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    fields: Final = _converse_bound_thinking_request(
+        {"anthropic-beta": "context-1m-2025-08-07"}, body_betas=("not-a-real-beta-2099",)
+    )
+    assert sorted(fields["anthropic_beta"]) == [
+        "context-1m-2025-08-07",
+        "thinking-binding-controls-2026-08-01",
+        "thinking-display-updates-2026-08-18",
+    ]

tests/unit/llms/bedrock/embed/test_bedrock_embedding.py

@@ -189,6 +189,290 @@ def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx.
     assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}]
 
 
+GOVCLOUD_WEST_RUNTIME: Final = "https://bedrock-runtime.us-gov-west-1.amazonaws.com"
+
+
+@pytest.mark.parametrize(
+    ("model_id", "embed_response"),
+    [
+        ("amazon.titan-embed-text-v2:0", titan_embedding_response),
+        (
+            "amazon.nova-2-multimodal-embeddings-v1:0",
+            {"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]},
+        ),
+        ("cohere.embed-english-v3", cohere_embedding_response),
+    ],
+)
+@pytest.mark.parametrize("sync_mode", [True, False])
+async def test_bedrock_embedding_calls_the_model_id_region_with_the_bare_model_id(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    model_id: str,
+    embed_response: dict[str, object],
+    sync_mode: bool,
+) -> None:
+    monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/{model_id}/invoke").mock(
+        return_value=httpx.Response(200, json=embed_response)
+    )
+    request: Final = {
+        "model": f"bedrock/us-gov-west-1/{model_id}",
+        "input": [test_input],
+        "api_key": "test-bearer-token",
+    }
+
+    response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request)
+
+    assert route.call_count == 1
+    assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
+
+
+def test_bedrock_embedding_prices_a_region_prefixed_model_from_its_region_row(
+    respx_mock: respx.MockRouter,
+) -> None:
+    model: Final = "bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"
+    respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
+        return_value=httpx.Response(200, json=titan_embedding_response)
+    )
+
+    response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token")
+
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens * litellm.model_cost[model]["input_cost_per_token"]
+    )
+
+
+def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region_for_routing_and_price(
+    respx_mock: respx.MockRouter,
+) -> None:
+    route = respx_mock.post(
+        "https://bedrock-runtime.us-gov-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke"
+    ).mock(return_value=httpx.Response(200, json=titan_embedding_response))
+
+    response: Final = litellm.embedding(
+        model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0",
+        input=[test_input],
+        api_key="test-bearer-token",
+        aws_region_name="us-gov-east-1",
+    )
+
+    assert route.call_count == 1
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens
+        * litellm.model_cost["bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
+    )
+
+
+def test_bedrock_embedding_model_id_arn_region_wins_over_the_model_path_region_and_env(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock(
+        return_value=httpx.Response(200, json=titan_embedding_response)
+    )
+
+    litellm.embedding(
+        model="bedrock/us-east-1/amazon.titan-embed-text-v2:0",
+        input=[test_input],
+        api_key="test-bearer-token",
+        model_id="arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.titan-embed-text-v2:0",
+    )
+
+    assert route.call_count == 1
+
+
+def test_bedrock_embedding_plain_model_id_calls_the_env_region_like_chat(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    route = respx_mock.post("https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.embed-english-v3/invoke").mock(
+        return_value=httpx.Response(200, json=cohere_embedding_response)
+    )
+
+    litellm.embedding(
+        model="bedrock/us-east-1/cohere.embed-english-v3",
+        input=[test_input],
+        api_key="test-bearer-token",
+        model_id="cohere.embed-english-v3",
+    )
+
+    assert route.call_count == 1
+
+
+def test_bedrock_embedding_rejects_an_empty_aws_region_name_instead_of_using_the_path_region(
+    respx_mock: respx.MockRouter,
+) -> None:
+    route = respx_mock.post(url__startswith=GOVCLOUD_WEST_RUNTIME).mock(
+        return_value=httpx.Response(200, json=titan_embedding_response)
+    )
+
+    with pytest.raises(litellm.APIConnectionError, match="Invalid AWS region format"):
+        litellm.embedding(
+            model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0",
+            input=[test_input],
+            api_key="test-bearer-token",
+            aws_region_name="",
+        )
+
+    assert route.call_count == 0
+
+
+@pytest.mark.parametrize(
+    "model,extra_params",
+    [
+        ("bedrock/invoke/us-gov-west-1/amazon.titan-embed-text-v2:0", {}),
+        ("bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", {"model_id": ""}),
+    ],
+)
+def test_bedrock_embedding_region_path_variants_call_the_path_region_with_the_bare_id_and_its_price(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    model: str,
+    extra_params: dict[str, str],
+) -> None:
+    monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
+    route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
+        return_value=httpx.Response(200, json=titan_embedding_response)
+    )
+
+    response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token", **extra_params)
+
+    assert route.call_count == 1
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens
+        * litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
+    )
+
+
+def test_bedrock_embedding_explicit_region_outside_the_known_list_returns_the_vector_at_the_bare_price(
+    respx_mock: respx.MockRouter,
+) -> None:
+    route = respx_mock.post(
+        "https://bedrock-runtime.zz-unlisted-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke"
+    ).mock(return_value=httpx.Response(200, json=titan_embedding_response))
+
+    response: Final = litellm.embedding(
+        model="bedrock/us-east-1/amazon.titan-embed-text-v2:0",
+        input=[test_input],
+        api_key="test-bearer-token",
+        aws_region_name="zz-unlisted-1",
+    )
+
+    assert route.call_count == 1
+    assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens * litellm.model_cost["amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
+    )
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+async def test_bedrock_embedding_without_a_region_path_bills_the_row_of_the_region_it_calls(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    sync_mode: bool,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock(
+        return_value=httpx.Response(200, json=titan_embedding_response)
+    )
+    request: Final = {
+        "model": "bedrock/amazon.titan-embed-text-v2:0",
+        "input": [test_input],
+        "api_key": "test-bearer-token",
+        "aws_region_name": "us-gov-west-1",
+    }
+
+    response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request)
+
+    assert route.call_count == 1
+    assert response.model == "amazon.titan-embed-text-v2:0"
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens
+        * litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"]
+    )
+
+
+def test_bedrock_cohere_embedding_bills_the_row_of_the_region_it_calls(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    regional_key: Final = "bedrock/us-gov-west-1/cohere.embed-english-v3"
+    monkeypatch.setitem(
+        litellm.model_cost,
+        regional_key,
+        {**litellm.model_cost["cohere.embed-english-v3"], "input_cost_per_token": 7e-07},
+    )
+    respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/cohere.embed-english-v3/invoke").mock(
+        return_value=httpx.Response(200, json=cohere_embedding_response)
+    )
+
+    response: Final = litellm.embedding(
+        model="bedrock/cohere.embed-english-v3",
+        input=[test_input],
+        api_key="test-bearer-token",
+        aws_region_name="us-gov-west-1",
+    )
+
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens * litellm.model_cost[regional_key]["input_cost_per_token"]
+    )
+
+
+def test_bedrock_embedding_with_a_govcloud_arn_model_id_bills_at_the_arn_region_row(
+    respx_mock: respx.MockRouter,
+) -> None:
+    arn: Final = "arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.nova-2-multimodal-embeddings-v1:0"
+    route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock(
+        return_value=httpx.Response(
+            200, json={"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]}
+        )
+    )
+
+    response: Final = litellm.embedding(
+        model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
+        input=[test_input],
+        api_key="test-bearer-token",
+        model_id=arn,
+    )
+
+    assert route.call_count == 1
+    assert response._hidden_params["response_cost"] == pytest.approx(
+        response.usage.prompt_tokens
+        * litellm.model_cost["bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0"]["input_cost_per_token"]
+    )
+
+
+@pytest.mark.parametrize(
+    "model",
+    [
+        "bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0",
+        "bedrock/async_invoke/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0",
+    ],
+)
+def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_that_region(
+    respx_mock: respx.MockRouter,
+    model: str,
+) -> None:
+    route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/async-invoke").mock(
+        return_value=httpx.Response(
+            200, json={"invocationArn": "arn:aws-us-gov:bedrock:us-gov-west-1:123456789012:async-invoke/abc123"}
+        )
+    )
+
+    litellm.embedding(
+        model=model,
+        input=[test_input],
+        api_key="test-bearer-token",
+        output_s3_uri="s3://test-bucket/async-invoke-output/",
+    )
+
+    assert route.call_count == 1
+    assert json.loads(route.calls[0].request.content)["modelId"] == "amazon.nova-2-multimodal-embeddings-v1:0"
+
+
 @pytest.mark.asyncio
 async def test_async_bedrock_embedding_with_bearer_token():
     """Test async embedding functionality with bearer token authentication"""

tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py

@@ -16,7 +16,7 @@ import httpx
 import pytest
 import requests_mock as requests_mock_lib
 import respx
-from pydantic import TypeAdapter
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 
 import litellm
 from litellm.constants import (
@@ -4200,6 +4200,98 @@ def test_bedrock_messages_without_guardrail_config_sends_no_guardrail_headers()
     assert not any(name.startswith("X-Amzn-Bedrock-Guardrail") for name in headers)
 
 
+@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
+def test_bedrock_messages_keeps_mid_conversation_output_config_its_auto_added_beta_gates() -> None:
+    """Invoke learns its body betas after the shared transform, so the gated field it supports must survive."""
+    from litellm.types.router import GenericLiteLLMParams
+
+    messages: Final = [
+        {"role": "user", "content": [{"type": "text", "text": "Hello"}]},
+        {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]},
+        {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}},
+        {"role": "user", "content": [{"type": "text", "text": "Go on"}]},
+    ]
+    result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
+        model="anthropic.claude-opus-4-8-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params={"max_tokens": 4096},
+        litellm_params=GenericLiteLLMParams(drop_params=True),
+        headers={},
+    )
+
+    assert "mid-conversation-output-config-2026-07-01" in result["anthropic_beta"]
+    assert {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}} in result["messages"]
+
+
+class _SentMessage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    role: str
+    content: list[dict[str, object]]
+
+
+_SENT_MESSAGES: Final = TypeAdapter(list[_SentMessage])
+
+
+@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
+@pytest.mark.parametrize("drop_params", [True, False])
+def test_bedrock_messages_drops_tool_changes_from_a_system_turn_an_older_model_gets_as_a_user_turn(
+    drop_params: bool, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A model without mid-conversation system support sends no tool-changes beta, so under drop_params the
+    converted turn must not carry the blocks that beta gates, and without it they go out as sent."""
+    monkeypatch.setattr(litellm, "drop_params", False)
+    from litellm.types.router import GenericLiteLLMParams
+
+    messages: Final = [
+        {"role": "user", "content": [{"type": "text", "text": "Hello"}]},
+        {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]},
+        {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "A weather tool is now available"},
+                {"type": "tool_addition", "tool": {"type": "tool_reference", "name": "get_weather"}},
+            ],
+        },
+        {"role": "user", "content": [{"type": "text", "text": "Go on"}]},
+    ]
+    result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
+        model="anthropic.claude-3-7-sonnet-20250219-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params={"max_tokens": 4096},
+        litellm_params=GenericLiteLLMParams(drop_params=drop_params),
+        headers={},
+    )
+
+    sent: Final = _SENT_MESSAGES.validate_python(result["messages"])
+    blocks: Final = [
+        block for message in sent for block in message.content
+    ]  # comprehension-ok: flattening the sent content blocks
+    assert "mid-conversation-tool-changes-2026-07-01" not in result.get("anthropic_beta", [])
+    assert any(block["type"] == "tool_addition" for block in blocks) is not drop_params
+    assert {"type": "text", "text": "A weather tool is now available"} in blocks
+    assert all(message.role != "system" for message in sent)
+
+
+@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
+@pytest.mark.parametrize("drop_params", (True, False))
+def test_bedrock_messages_sends_the_binding_beta_with_block_binding_whatever_drop_params(drop_params: bool) -> None:
+    from litellm.types.router import GenericLiteLLMParams
+
+    result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
+        model="us.anthropic.claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "Reply with OK"}],
+        anthropic_messages_optional_request_params={
+            "max_tokens": 2048,
+            "thinking": {"type": "enabled", "budget_tokens": 1024, "block_binding": BLOCK_BINDING},
+        },
+        litellm_params=GenericLiteLLMParams(drop_params=drop_params),
+        headers={},
+    )
+
+    assert result["thinking"]["block_binding"] == BLOCK_BINDING
+    assert "thinking-binding-controls-2026-08-01" in result["anthropic_beta"]
+
+
 @dataclass(slots=True)
 class BedrockDownAnthropicUp:
     anthropic_bodies: tuple[dict[str, object], ...] = ()