chore: merge main and keep client prompt cache breakpoints on routes that do not translate them

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -6,12 +6,13 @@
 
 ## Handwritten 1-3 sentence description
 
-<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
-     plain sentences proves the author understands what it does, and human-written technical English is far easier
-     for a reviewer to parse than AI-written English
-     An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
-     backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
-     Tags still present at review time mean the summary is AI-written and has not been rewritten yet
+<!-- The author writes a plain summary in their own words before requesting review.
+     Wrap newly AI-written summary text in the make-me-human-readable tags below, including backticks.
+     The author reviews or rewrites the draft and removes the tags.
+     When updating an existing PR, preserve the author's wording and removed tags.
+     Changes elsewhere in the body do not make an unchanged summary an AI draft again.
+     Add tags again only around newly AI-written replacement summary text, never merely to match this template.
+     This preserves author review across later updates (2026-11-02, #49779).
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
      No bullets, tables, or code names here. Everything else goes inside the Details block below -->
 

.github/workflows/test-unit.yml

@@ -441,6 +441,7 @@ jobs:
               tests/unit/ocr
               tests/unit/passthrough
               tests/unit/rag
+              tests/unit/cookbook
               tests/unit/realtime_api
               tests/unit/rerank_api
               tests/unit/search

README.md

@@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) |  |  |  | ✅ |  |  |  |  |  |  |
 | [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) |  |  |  |  |  |  |  |  |  |  |
 | [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

cookbook/gpt_realtime_translate.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+import sys
+import wave
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlencode, urlsplit, urlunsplit
+
+import websockets
+from websockets.asyncio.client import ClientConnection
+
+SAMPLE_RATE = 24_000
+CHANNELS = 1
+SAMPLE_WIDTH = 2
+CHUNK_DURATION_SECONDS = 0.1
+CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
+RESPONSE_TIMEOUT_SECONDS = 30.0
+AUDIO_EVENT_TYPES = frozenset(
+    {
+        "session.output_audio.delta",
+        "response.audio.delta",
+        "response.output_audio.delta",
+    }
+)
+TRANSCRIPT_EVENT_TYPES = frozenset(
+    {
+        "session.output_transcript.delta",
+        "response.text.delta",
+        "response.output_audio_transcript.delta",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    input_wav: Path
+    output_wav: Path
+    base_url: str
+    model: str
+    target_language: str
+    trailing_silence_seconds: float
+    api_key: str
+
+
+def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
+    sys.stdout.write(f"{message}{end}")
+    if flush:
+        sys.stdout.flush()
+
+
+def write_stderr(message: str) -> None:
+    sys.stderr.write(f"{message}\n")
+
+
+def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
+    parser = argparse.ArgumentParser(
+        description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
+    )
+    parser.add_argument("input_wav", type=Path)
+    parser.add_argument("--output", type=Path, default=Path("translated.wav"))
+    parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
+    parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
+    parser.add_argument("--target-language", default="fr")
+    parser.add_argument("--trailing-silence", type=float, default=1.5)
+    parsed = parser.parse_args(argv)
+    api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
+    if parsed.trailing_silence < 0:
+        return "--trailing-silence must be zero or greater"
+    return Settings(
+        input_wav=parsed.input_wav,
+        output_wav=parsed.output,
+        base_url=parsed.base_url,
+        model=parsed.model,
+        target_language=parsed.target_language,
+        trailing_silence_seconds=parsed.trailing_silence,
+        api_key=api_key,
+    )
+
+
+def translation_url(base_url: str, model: str) -> str | None:
+    parsed = urlsplit(base_url.rstrip("/"))
+    scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
+    if not scheme or not parsed.netloc:
+        return None
+    base_path = parsed.path.rstrip("/")
+    realtime_path = (
+        f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
+    )
+    return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
+
+
+def read_pcm16_wav(path: Path) -> bytes | str:
+    try:
+        with wave.open(str(path), "rb") as source:
+            actual_format = (
+                source.getnchannels(),
+                source.getsampwidth(),
+                source.getframerate(),
+                source.getcomptype(),
+            )
+            expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
+            if actual_format != expected_format:
+                return (
+                    f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
+                    f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
+                    f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
+                )
+            return source.readframes(source.getnframes())
+    except (OSError, EOFError, wave.Error) as exc:
+        return f"Unable to read {path}: {exc}"
+
+
+def audio_chunks(audio: bytes) -> Iterator[bytes]:
+    return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
+
+
+def audio_message(audio: bytes) -> str:
+    return json.dumps(
+        {
+            "type": "session.input_audio_buffer.append",
+            "audio": base64.b64encode(audio).decode("ascii"),
+        }
+    )
+
+
+async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
+    await connection.send(
+        json.dumps(
+            {
+                "type": "session.update",
+                "session": {"audio": {"output": {"language": target_language}}},
+            }
+        )
+    )
+    while True:
+        raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
+        event = json.loads(raw_event)
+        event_type = event.get("type")
+        if event_type == "session.created":
+            write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
+        if event_type == "session.updated":
+            return None
+        if event_type == "error":
+            return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
+
+
+async def send_audio(
+    connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
+) -> None:
+    silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
+    try:
+        for chunk in audio_chunks(pcm + silence):
+            await connection.send(audio_message(chunk))
+            await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
+        await connection.send(json.dumps({"type": "session.close"}))
+    finally:
+        finished.set()
+
+
+async def receive_translation(
+    connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
+) -> str | None:
+    audio_received = asyncio.Event()
+    try:
+        with wave.open(str(output_path), "wb") as output:
+            output.setnchannels(CHANNELS)
+            output.setsampwidth(SAMPLE_WIDTH)
+            output.setframerate(SAMPLE_RATE)
+            write_stdout("Translation: ", end="", flush=True)
+            while True:
+                try:
+                    raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
+                except TimeoutError:
+                    if not sender_finished.is_set():
+                        continue
+                    return (
+                        f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
+                    )
+                event = json.loads(raw_event)
+                event_type = event.get("type")
+                if event_type in AUDIO_EVENT_TYPES:
+                    output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
+                    audio_received.set()
+                elif event_type in TRANSCRIPT_EVENT_TYPES:
+                    write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
+                elif event_type == "error":
+                    return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
+                elif event_type == "session.closed":
+                    write_stdout()
+                    return None if audio_received.is_set() else "The translation stream ended without translated audio"
+    except (OSError, wave.Error) as exc:
+        return f"Unable to write {output_path}: {exc}"
+
+
+async def translate(settings: Settings, pcm: bytes) -> str | None:
+    url = translation_url(settings.base_url, settings.model)
+    if not url:
+        return f"Invalid --base-url: {settings.base_url}"
+    sender_finished = asyncio.Event()
+    try:
+        async with websockets.connect(
+            url,
+            additional_headers={"Authorization": f"Bearer {settings.api_key}"},
+            proxy=None,
+            open_timeout=20,
+            close_timeout=5,
+        ) as connection:
+            configuration_error = await configure_session(connection, settings.target_language)
+            if configuration_error:
+                return configuration_error
+            async with asyncio.TaskGroup() as tasks:
+                receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
+                tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
+            return receiver.result()
+    except Exception as exc:
+        return f"Translation failed: {type(exc).__name__}: {exc}"
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    settings = parse_args(argv)
+    if isinstance(settings, str):
+        write_stderr(settings)
+        return 2
+    pcm = read_pcm16_wav(settings.input_wav)
+    if isinstance(pcm, str):
+        write_stderr(pcm)
+        return 2
+    error = asyncio.run(translate(settings, pcm))
+    if error:
+        write_stderr(error)
+        return 1
+    write_stdout(f"Translated audio: {settings.output_wav}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -3,9 +3,11 @@
 
 import asyncio
 import base64
+import copy
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from itertools import chain
 from types import MappingProxyType
@@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
 
 import litellm
 from litellm import Router, verbose_logger
-from litellm._internal_context import with_service_target
+from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.constants import (
@@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
     request_tags_from_metadata,
 )
 from litellm.repositories.managed_file_repository import ManagedFileRepository
+from litellm.responses.streaming_iterator import spawn_logging_task
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccessIssue]
     AllMessageValues,
@@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
+_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
+_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderUpload:
+    model_id: str
+    provider_file_id: str
+    upload_call_id: str | None = None
+
+
+def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
+    file_hidden_params: Final = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
+        dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
+    )
+    model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
+        file_hidden_params.get("model_file_id_mapping") or {}
+    )
+    upload_call_id: Final = file_hidden_params.get("litellm_call_id")
+    return tuple(
+        _ProviderUpload(
+            model_id=model_id,
+            provider_file_id=provider_file_id,
+            upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
+        )
+        for model_id, provider_file_id in model_file_id_mapping.items()
+    )
+
+
+def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
+    credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
+    return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
+
+
+def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
+    upload: Final = uploads[index]
+    other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
+    return all(other.model_id != upload.model_id for other in other_uploads)
+
+
+def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
+    trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
+    provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
+    return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
+
+
+def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
+    decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
+    carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
+    if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
+        return None
+    return carried
+
+
+async def _delete_provider_file(
+    llm_router: Router,
+    model_id: str,
+    provider_file_id: str,
+    request_data: Mapping[str, object],
+) -> None:
+    credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
+    delete_data: Final = {
+        **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
+        **(
+            {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
+            if credentials is not None
+            else {}
+        ),
+    }
+    await llm_router.afile_delete(
+        model=model_id,
+        file_id=provider_file_id,
+        **cast(_RouterFileCallKwargs, delete_data),
+    )
+
+
+def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
+    return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
+
+
+async def _delete_redundant_upload(
+    llm_router: Router,
+    upload: _ProviderUpload,
+    delete_request: Mapping[str, object],
+    parent_call_id: str | None,
+) -> None:
+    try:
+        with internal_sub_call(parent_call_id):
+            await _delete_provider_file(
+                llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
+            )
+    except Exception as error:
+        verbose_logger.warning(
+            f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
+            f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
+            f"{type(error).__name__} {sanitize_for_log(error)}"
+        )
+
+
+async def _delete_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    delete_request: Mapping[str, object],
+) -> None:
+    await asyncio.gather(
+        *(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
+    )
+
+
+def _start_deleting_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    create_file_request: CreateFileRequest,
+) -> None:
+    if not uploads:
+        return
+    delete_request: Final = MappingProxyType(
+        {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
+    )
+    with post_response_phase():
+        spawn_logging_task(
+            asyncio.get_running_loop(),
+            _delete_redundant_uploads(llm_router, uploads, delete_request),
+            task_name="managed file repeat upload cleanup",
+        )
+
+
 def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
     return prisma_client.db
 
@@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             target_model_names_list=target_model_names_list,
         )
 
-        ## STORE MODEL MAPPINGS IN DB
-        model_mappings: Dict[str, str] = {}
-
-        for file_object in responses:
-            file_hidden_params = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
-                dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
-            )
-            model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
-            if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
-                model_mappings.update(model_file_id_mapping)
+        uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
+        per_target_model_ids: Final = frozenset(
+            model_id
+            for model_id in {upload.model_id for upload in uploads}
+            if _keeps_one_upload_per_target(llm_router, model_id)
+        )
+        kept_uploads: Final = tuple(
+            upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
+        )
+        model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
 
         await self.store_unified_file_id(
             file_id=response.id,
@@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             model_mappings=model_mappings,
             user_api_key_dict=user_api_key_dict,
         )
+        _start_deleting_redundant_uploads(
+            llm_router,
+            tuple(
+                upload
+                for upload in uploads
+                if upload not in kept_uploads and upload.model_id not in per_target_model_ids
+            ),
+            create_file_request,
+        )
 
         # Emit Prometheus metrics for managed file creation
         prom_logger = self._get_prometheus_logger()
@@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
         if not specific_model_file_id_mapping:
             return
-        filtered_data: Final = {
-            k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
-        }
         for model_id, model_file_id in specific_model_file_id_mapping.items():
-            credentials = llm_router.registry().credentials(model_id, form="with_provider")
-            delete_data = {
-                **filtered_data,
-                **(
-                    {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
-                    if credentials is not None
-                    else {}
-                ),
-            }
-            await llm_router.afile_delete(
-                model=model_id,
-                file_id=model_file_id,
-                **cast(_RouterFileCallKwargs, delete_data),
+            await _delete_provider_file(llm_router, model_id, model_file_id, data)
+        unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
+        delete_call_id: Final = data.get("litellm_call_id")
+        if unrecorded_upload is not None:
+            await _delete_redundant_upload(
+                llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
             )
 
     async def afile_content(

litellm-rust/crates/callbacks-legacy-python/python_contract.json

@@ -33,6 +33,17 @@
     "api_key",
     "additional_args"
   ],
+  "record_response": [
+    "logger",
+    "status",
+    "headers",
+    "body"
+  ],
+  "record_api_window": [
+    "logger",
+    "start",
+    "end"
+  ],
   "defers_async_logging": [
     "logger"
   ],
@@ -91,6 +102,13 @@
     "logger",
     "head"
   ],
+  "stamp_stream_head": [
+    "head",
+    "logger",
+    "kwargs",
+    "start_time",
+    "end_time"
+  ],
   "stream_success": [
     "logger",
     "url_route",

litellm-rust/crates/callbacks-legacy-python/src/adapter.rs

@@ -57,6 +57,8 @@ pub struct LegacyLogging {
     asynchronous: bool,
     internal: bool,
     cache_key: Option<String>,
+    /// When the provider request went out, until its window is recorded (Messages only).
+    api_started: Option<f64>,
 }
 
 fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
@@ -87,6 +89,60 @@ impl LegacyLogging {
             asynchronous,
             internal: false,
             cache_key: None,
+            api_started: None,
+        }
+    }
+
+    /// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as
+    /// its own argument, so the logger never sees the caller's raw metadata, only the validated
+    /// request metadata merged in through `litellm_params`. Python updates the logger once, before
+    /// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the
+    /// logged metadata is not rebuilt from what the call wrote into it since.
+    fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult<Bound<'py, PyDict>> {
+        let kwargs = self.call.kwargs().bind(py).copy()?;
+        if self.operation != LoggingOperation::Messages {
+            return Ok(kwargs);
+        }
+        let consumed: &[&str] = if refresh {
+            &["metadata", "litellm_metadata"]
+        } else {
+            &["metadata"]
+        };
+        for name in consumed {
+            if kwargs.contains(name)? {
+                kwargs.del_item(name)?;
+            }
+        }
+        Ok(kwargs)
+    }
+
+    /// Python's Messages handler hands the logger its request params, validated `metadata`
+    /// included, as `litellm_params`, beside the `aanthropic_messages` call flag.
+    fn logged_request_params(
+        &self,
+        context: &RequestContext,
+    ) -> Option<serde_json::Map<String, Value>> {
+        if self.operation != LoggingOperation::Messages {
+            return None;
+        }
+        let mut params = context
+            .optional_params
+            .as_object()
+            .cloned()
+            .unwrap_or_default();
+        params.insert(
+            "aanthropic_messages".to_owned(),
+            Value::Bool(self.asynchronous),
+        );
+        Some(params)
+    }
+
+    /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync
+    /// client is untimed.
+    fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> {
+        match self.api_started.take() {
+            Some(start) => self.logger()?.record_api_window(py, start, end),
+            None => Ok(()),
         }
     }
 
@@ -111,6 +167,16 @@ impl LegacyLogging {
         }
     }
 
+    /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key.
+    fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> {
+        match self.operation {
+            LoggingOperation::Messages => Some(""),
+            LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => {
+                context.api_key.as_ref().map(|api_key| api_key.expose())
+            }
+        }
+    }
+
     fn stream_billing(&self) -> Option<PassThroughStream> {
         match self.operation {
             LoggingOperation::Messages => Some(PassThroughStream {
@@ -377,7 +443,13 @@ impl LegacyLogging {
         context: &RequestContext,
     ) -> PyResult<HookStep<Self, Box<WireRequest>>> {
         let logger = self.logger()?;
-        logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?;
+        logger.update_from_kwargs(
+            py,
+            &self.logging_kwargs(py, false)?,
+            &wire,
+            context,
+            self.logged_request_params(context).as_ref(),
+        )?;
         let body = to_py(py, &wire.body)?
             .into_bound(py)
             .cast_into::<PyDict>()?;
@@ -400,11 +472,14 @@ impl LegacyLogging {
         self.logger()?.pre_call(
             py,
             self.input_description(),
-            context.api_key.as_ref().map(|api_key| api_key.expose()),
+            self.logged_api_key(context),
             &body,
             &headers,
             &wire.url,
         )?;
+        if self.operation == LoggingOperation::Messages && self.asynchronous {
+            self.api_started = Some(epoch_seconds());
+        }
         let headers = headers
             .iter()
             .map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
@@ -455,7 +530,7 @@ impl LegacyLogging {
             py,
             (
                 &logger,
-                self.call.kwargs(),
+                self.logging_kwargs(py, true)?,
                 &facts.provider.model,
                 logger.getattr("optional_params")?,
                 params,
@@ -483,6 +558,16 @@ impl LegacyLogging {
         py: Python<'_>,
         raw: &RawResponse,
     ) -> PyResult<HookStep<Self, ()>> {
+        if self.operation == LoggingOperation::Messages {
+            // Python's Messages handler never calls `post_call` at receipt: it stores the raw
+            // response, and logging replays it through `AnthropicConfig.transform_response`.
+            self.close_api_window(py, epoch_seconds())?;
+            if let Some(head) = &raw.head {
+                self.logger()?
+                    .record_response(py, head.status, &head.headers, &raw.body)?;
+            }
+            return Ok(HookStep::Ready(()));
+        }
         let api_key = self
             .request
             .as_ref()
@@ -522,6 +607,9 @@ impl LegacyLogging {
     ) -> PyResult<HookStep<Self, ()>> {
         self.end = Some(datetime(py, timing.end_time)?);
         self.error = Some(error.clone_ref(py).into_value(py));
+        if self.logger.is_some() {
+            self.close_api_window(py, timing.end_time)?;
+        }
         if self.stream.is_some() {
             return self.stream_failure(py);
         }
@@ -543,6 +631,20 @@ impl LegacyLogging {
             head.bind(py).set_item("cache_key", key)?;
             head.bind(py).set_item("cache_hit", true)?;
         }
+        self.close_api_window(py, epoch_seconds())?;
+        if self.operation == LoggingOperation::Messages {
+            // Python's `@client` stamps the call metadata onto the stream it hands back.
+            Streaming::StampHead.call(
+                py,
+                (
+                    head.bind(py),
+                    self.logger()?.object(py),
+                    self.call.kwargs(),
+                    &self.start,
+                    datetime(py, epoch_seconds())?,
+                ),
+            )?;
+        }
         Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
         self.stream = Some(DeliveredStream {
             chunks: PyList::empty(py).unbind(),
@@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update
         let step = logging
             .before_provider_request(py, Box::new(wire), context)
             .unwrap();
-        let raw = RawResponse {
-            body: "raw response".into(),
-        };
+        let raw = RawResponse::new("raw response".into());
         assert!(matches!(
             logging
                 .on_event(
@@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response
                 py,
                 &locals,
                 c"
-assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
+assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls
 assert logger.calls[0][1] is head
-chunks = logger.calls[1][1]
+assert logger.calls[1][1] is head
+chunks = logger.calls[2][1]
 assert len(chunks) == 2
 assert chunks[0] is first
 assert chunks[1] is last

litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs

@@ -5,6 +5,7 @@
 use litellm_host::interceptors::{RequestContext, WireRequest};
 use litellm_host_python::to_py;
 use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
+use serde_json::Value;
 
 use crate::logger::PythonLogger;
 use crate::python::{Logging, Wrapper};
@@ -15,9 +16,10 @@ pub trait LegacyCallbacks {
     fn update_from_kwargs(
         &self,
         py: Python<'_>,
-        kwargs: &Py<PyDict>,
+        kwargs: &Bound<'_, PyDict>,
         wire: &WireRequest,
         context: &RequestContext,
+        request_params: Option<&serde_json::Map<String, Value>>,
     ) -> PyResult<()>;
 
     /// `Logging.pre_call`.
@@ -31,6 +33,20 @@ pub trait LegacyCallbacks {
         url: &str,
     ) -> PyResult<()>;
 
+    /// Records the provider's raw HTTP response on the logger, as Python's Messages handler
+    /// does with `httpx_response` and `response_headers`.
+    fn record_response(
+        &self,
+        py: Python<'_>,
+        status: u16,
+        headers: &[(String, String)],
+        body: &str,
+    ) -> PyResult<()>;
+
+    /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post`
+    /// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`.
+    fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>;
+
     /// `Logging.post_call`.
     fn post_call(
         &self,
@@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger {
     fn update_from_kwargs(
         &self,
         py: Python<'_>,
-        kwargs: &Py<PyDict>,
+        kwargs: &Bound<'_, PyDict>,
         wire: &WireRequest,
         context: &RequestContext,
+        request_params: Option<&serde_json::Map<String, Value>>,
     ) -> PyResult<()> {
         let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect();
-        let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?;
+        let redacted_kwargs = redact(py, kwargs, &secret_fields)?;
         let optional_params = redact(
             py,
             &to_py(py, &context.optional_params)?
@@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger {
             &secret_fields,
         )?;
         let params = PyDict::new(py);
-        params.set_item(
-            "litellm_call_id",
-            kwargs.bind(py).get_item("litellm_call_id")?,
-        )?;
+        params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
         params.set_item("api_base", &wire.url)?;
         for name in ["logger_fn", "litellm_request_debug"] {
-            if let Some(value) = kwargs.bind(py).get_item(name)? {
+            if let Some(value) = kwargs.get_item(name)? {
                 params.set_item(name, value)?;
             }
         }
         for name in custom_pricing_fields(py)? {
-            if let Some(value) = kwargs.bind(py).get_item(&name)?
+            if let Some(value) = kwargs.get_item(&name)?
                 && !value.is_none()
             {
                 params.set_item(name, value)?;
             }
         }
+        for (name, value) in request_params.into_iter().flatten() {
+            params.set_item(name, to_py(py, value)?)?;
+        }
         Logging::Update.call(
             py,
             (
@@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger {
         Ok(())
     }
 
+    fn record_response(
+        &self,
+        py: Python<'_>,
+        status: u16,
+        headers: &[(String, String)],
+        body: &str,
+    ) -> PyResult<()> {
+        Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?;
+        Ok(())
+    }
+
+    fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> {
+        Logging::RecordApiWindow.call(py, (self.object(py), start, end))?;
+        Ok(())
+    }
+
     fn post_call(
         &self,
         py: Python<'_>,

litellm-rust/crates/callbacks-legacy-python/src/python.rs

@@ -44,6 +44,10 @@ pub(crate) enum Logging {
     PreCall,
     #[strum(serialize = "post_call")]
     PostCall,
+    #[strum(serialize = "record_response")]
+    RecordResponse,
+    #[strum(serialize = "record_api_window")]
+    RecordApiWindow,
     #[strum(serialize = "defers_async_logging")]
     DefersAsync,
     #[strum(serialize = "defer_success")]
@@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks {
 pub(crate) enum Streaming {
     #[strum(serialize = "stream_opened")]
     Opened,
+    #[strum(serialize = "stamp_stream_head")]
+    StampHead,
     #[strum(serialize = "stream_success")]
     Success,
     #[strum(serialize = "stream_failure")]

litellm-rust/crates/callbacks-legacy-python/src/test_support.rs

@@ -65,6 +65,12 @@ FAKES = {
     'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call(
         original_response, api_key, additional_args
     ),
+    'record_response': lambda logger, status, headers, body: logger.model_call_details.update(
+        response_headers=dict(headers)
+    ),
+    'record_api_window': lambda logger, start, end: logger.model_call_details.update(
+        llm_api_duration_ms=(end - start) * 1000
+    ),
     'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)),
     'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending),
     'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls(
@@ -85,6 +91,7 @@ FAKES = {
     ),
     'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
     'stream_opened': lambda logger, head: logger.record('stream_opened', head),
+    'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head),
     'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
         'stream_success', list(chunks)
     ),

litellm-rust/crates/host-http/tests/serve.rs

@@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(Bytes::from(wire.url)))
         },
@@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once(
         move |_, _, route_hooks, _observations| async move {
             if event {
                 route_hooks
-                    .after_provider_response(RawResponse {
-                        body: "response".into(),
-                    })
+                    .after_provider_response(RawResponse::new("response".into()))
                     .await?;
             } else {
                 route_hooks
@@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(json!({"url": wire.url})))
         },
@@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting(
         None,
         |_, _, route_hooks, _observations| async move {
             route_hooks
-                .after_provider_response(RawResponse { body: "raw".into() })
+                .after_provider_response(RawResponse::new("raw".into()))
                 .await?;
             Err(TestError::Provider)
         },

litellm-rust/crates/host-native/tests/driver.rs

@@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine {
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(wire.url))
         },
@@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re
         "input",
         Some(sender),
         |_, _, interceptors, observers| async move {
-            let raw = RawResponse {
-                body: "provider response".into(),
-            };
+            let raw = RawResponse::new("provider response".into());
             observers.unwrap().emit(CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
             ));

litellm-rust/crates/host-python/tests/support/mod.rs

@@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory {
                     .before_provider_request(wire(), context())
                     .await?;
                 host.interceptors
-                    .after_provider_response(RawResponse { body: "raw".into() })
+                    .after_provider_response(RawResponse::new("raw".into()))
                     .await?;
                 Ok(format!("{projected}|{signed}|{}", wire.url))
             })

litellm-rust/crates/host/src/interceptors.rs

@@ -28,6 +28,28 @@ pub struct RequestContext {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct RawResponse {
     pub body: String,
+    /// The provider's HTTP status and headers, when the route forwards them to logging.
+    pub head: Option<ResponseHead>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ResponseHead {
+    pub status: u16,
+    pub headers: Vec<(String, String)>,
+}
+
+impl RawResponse {
+    pub fn new(body: String) -> Self {
+        Self { body, head: None }
+    }
+
+    #[must_use]
+    pub fn with_head(self, head: ResponseHead) -> Self {
+        Self {
+            head: Some(head),
+            ..self
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -186,7 +208,7 @@ mod tests {
                 .await?;
                 Interceptors::after_provider_response(
                     &channel.interceptors,
-                    RawResponse { body: "raw".into() },
+                    RawResponse::new("raw".into()),
                 )
                 .await?;
                 Ok((sent, ()))

litellm-rust/crates/host/tests/observation.rs

@@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile
 
 #[rstest]
 fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) {
-    let mut raw = RawResponse {
-        body: "provider response".into(),
-    };
+    let mut raw = RawResponse::new("provider response".into());
     let event: CallEvent<(), (), &RawResponse> =
         CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw });
     let expected = raw.clone();

litellm-rust/crates/http/src/error.rs

@@ -58,8 +58,20 @@ pub(crate) enum Repr {
     Network(#[source] reqwest::Error),
     #[error("upstream request timed out")]
     Timeout(#[source] reqwest::Error),
+    /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says
+    /// the response head had already arrived.
+    #[error("upstream request timed out")]
+    ReadTimeout {
+        #[source]
+        elapsed: Elapsed,
+        body: bool,
+    },
     #[error("upstream request failed with status {status}: {body}")]
-    Status { status: u16, body: UpstreamBody },
+    Status {
+        status: u16,
+        body: UpstreamBody,
+        headers: Vec<(String, String)>,
+    },
 
     #[error("invalid WebSocket URL")]
     WebSocketUrl(#[source] url::ParseError),
@@ -136,13 +148,59 @@ impl Repr {
 
 impl Error {
     pub fn upstream(status: u16, body: String) -> Self {
+        Self::upstream_with_headers(status, body, Vec::new())
+    }
+
+    /// An upstream failure that keeps the provider's response headers (`retry-after`,
+    /// `request-id`) for the caller.
+    pub fn upstream_with_headers(
+        status: u16,
+        body: String,
+        headers: Vec<(String, String)>,
+    ) -> Self {
         Repr::Status {
             status,
             body: UpstreamBody(body),
+            headers,
         }
         .into()
     }
 
+    /// The provider's response headers on an upstream failure; empty for every other error.
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match &*self.0 {
+            Repr::Status { headers, .. } => headers,
+            _ => &[],
+        }
+    }
+
+    /// A timeout reached before the connection was established, as opposed to while reading.
+    pub fn is_connect_timeout(&self) -> bool {
+        matches!(&*self.0, Repr::Timeout(error) if error.is_connect())
+    }
+
+    /// A read the caller timed itself (httpx's per-read timeout) ran out.
+    pub fn read_timeout(elapsed: Elapsed) -> Self {
+        Repr::ReadTimeout {
+            elapsed,
+            body: false,
+        }
+        .into()
+    }
+
+    /// A read of the response body the caller timed itself ran out, after the head arrived.
+    pub fn body_read_timeout(elapsed: Elapsed) -> Self {
+        Repr::ReadTimeout {
+            elapsed,
+            body: true,
+        }
+        .into()
+    }
+
+    pub fn is_body_read_timeout(&self) -> bool {
+        matches!(&*self.0, Repr::ReadTimeout { body: true, .. })
+    }
+
     pub fn computed_header(name: impl Into<String>) -> Self {
         Repr::ComputedHeader(name.into()).into()
     }
@@ -169,6 +227,7 @@ impl Error {
         match &*self.0 {
             Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::Handshake(_)
             | Repr::HandshakeTimeout(_)
@@ -227,6 +286,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::Handshake(_)
             | Repr::HandshakeTimeout(_)
@@ -259,6 +319,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -295,6 +356,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -333,6 +395,7 @@ impl Error {
             | Repr::Signature(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -356,7 +419,10 @@ impl Error {
 
     pub fn is_timeout(&self) -> bool {
         match &*self.0 {
-            Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true,
+            Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
+            | Repr::HandshakeTimeout(_)
+            | Repr::DownloadTimeout(_) => true,
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -392,7 +458,7 @@ impl Error {
 
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match &*self.0 {
-            Repr::Status { status, body } => Some((*status, &body.0)),
+            Repr::Status { status, body, .. } => Some((*status, &body.0)),
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -408,6 +474,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
             | Repr::WebSocketSchemeRewrite(_)
@@ -431,7 +498,7 @@ impl Error {
     pub fn status(&self) -> Option<u16> {
         match &*self.0 {
             Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status),
-            Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408),
+            Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408),
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -482,6 +549,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)

litellm-rust/crates/inference-chat/src/handler.rs

@@ -108,7 +108,7 @@ pub(super) async fn execute(
             if !status.is_success() {
                 return Err(litellm_http::Error::upstream(status.as_u16(), text).into());
             }
-            let raw = RawResponse { body: text.clone() };
+            let raw = RawResponse::new(text.clone());
             if let Some(observers) = observers {
                 observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                     ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference-messages/Cargo.toml

@@ -24,6 +24,7 @@ litellm-tracing.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
+tokio.workspace = true
 tracing.workspace = true
 
 [dev-dependencies]
@@ -34,5 +35,4 @@ litellm-host-native.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }
 litellm-inference-testing.workspace = true
 rstest.workspace = true
-tokio.workspace = true
 wiremock.workspace = true

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -35,6 +35,14 @@ impl MessagesProvider {
     pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
         self.config
     }
+
+    /// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped.
+    pub(crate) fn accepts_service_tier(self) -> bool {
+        matches!(
+            self.provider,
+            LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle
+        )
+    }
 }
 
 /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
@@ -119,3 +127,69 @@ mod tests {
         );
     }
 }
+
+/// Python's `_merge_messages_header_sources`: later sources win per header name, except
+/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and
+/// sorted into one lowercase `anthropic-beta` entry.
+pub fn merge_messages_header_sources(
+    sources: impl IntoIterator<Item = Map<String, Value>>,
+) -> Map<String, Value> {
+    let mut merged = Map::new();
+    let mut betas = std::collections::BTreeSet::new();
+    for (name, value) in sources.into_iter().flatten() {
+        if !name.eq_ignore_ascii_case("anthropic-beta") {
+            merged.insert(name, value);
+            continue;
+        }
+        let values: Vec<&str> = match &value {
+            Value::String(text) => vec![text.as_str()],
+            Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(),
+            _ => Vec::new(),
+        };
+        betas.extend(
+            values
+                .iter()
+                .flat_map(|value| value.split(','))
+                .map(str::trim)
+                .filter(|beta| !beta.is_empty())
+                .map(str::to_owned),
+        );
+    }
+    if !betas.is_empty() {
+        merged.insert(
+            "anthropic-beta".to_owned(),
+            Value::String(betas.into_iter().collect::<Vec<_>>().join(",")),
+        );
+    }
+    merged
+}
+
+#[cfg(test)]
+mod merge_tests {
+    use serde_json::{Map, Value, json};
+
+    use super::merge_messages_header_sources;
+
+    fn map(value: Value) -> Map<String, Value> {
+        value.as_object().cloned().unwrap()
+    }
+
+    #[test]
+    fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() {
+        let merged = merge_messages_header_sources([
+            map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})),
+            map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})),
+            map(json!({"anthropic-beta": 5})),
+        ]);
+        assert_eq!(
+            Value::Object(merged),
+            json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"})
+        );
+    }
+
+    #[test]
+    fn no_beta_values_leave_no_beta_header() {
+        let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]);
+        assert!(merged.is_empty());
+    }
+}

litellm-rust/crates/inference-messages/src/handler.rs

@@ -10,9 +10,9 @@ use litellm_core_utils::{
     get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
 };
 use litellm_host::interceptors::{
-    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
+    ResponseHead, ResultSource, WireRequest,
 };
-use litellm_http::request::with_default_headers;
 use litellm_llms::base_llm::{
     auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
@@ -21,14 +21,16 @@ use litellm_llms::base_llm::{
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
     Error, MessagesCall, MessagesCallResponse, MessagesRoute,
-    common_utils::{MessagesProvider, string_headers},
+    common_utils::{MessagesProvider, merge_messages_header_sources, string_headers},
     constants::MESSAGES_TIMEOUT_SECS,
     route::Messages,
     types::invalid_request,
@@ -126,14 +128,18 @@ async fn provider_call(
         MessagesRequest { model, ..body },
         shaping.settings.reasoning_auto_summary,
     )?;
+    let transform_context =
+        MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params);
+    let filtered = requested_optional_params(sanitized, provider, &transform_context)?;
     let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+        without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?;
 
     let scoped =
         get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
+    let forwarded = string_headers(Some(merge_messages_header_sources([
+        extra_headers.unwrap_or_default(),
+        scoped,
+    ])))?;
     let validated = config.validate_environment(
         forwarded,
         api_key.as_deref(),
@@ -144,16 +150,13 @@ async fn provider_call(
     )?;
 
     let body = config.request_body(
-        config.transform_anthropic_messages_request(
-            trimmed,
-            &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-        )?,
+        config.transform_anthropic_messages_request(trimmed, &transform_context)?,
         &validated.headers,
         shaping.settings.drop_params,
     )?;
     let environment = ValidatedEnvironment {
         headers: config.request_headers(
-            with_default_headers(validated.headers, config.default_headers()),
+            with_python_default_headers(validated.headers, config.default_headers()),
             &body,
         ),
         auth: validated.auth,
@@ -184,7 +187,7 @@ async fn provider_call(
         .before_provider_request(
             WireRequest {
                 url,
-                headers: authenticated.headers,
+                headers: joined_header_values(authenticated.headers),
                 body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
             request_context,
@@ -213,6 +216,64 @@ async fn provider_call(
     })
 }
 
+/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in
+/// case into one lowercase header whose values are joined with `", "`.
+fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> {
+    let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len());
+    let mut positions: std::collections::HashMap<String, usize> =
+        std::collections::HashMap::with_capacity(headers.len());
+    for (name, value) in headers {
+        match positions.entry(name.to_ascii_lowercase()) {
+            std::collections::hash_map::Entry::Occupied(position) => {
+                let present = &mut joined[*position.get()].1;
+                present.push_str(", ");
+                present.push_str(&value);
+            }
+            std::collections::hash_map::Entry::Vacant(position) => {
+                position.insert(joined.len());
+                joined.push((name, value));
+            }
+        }
+    }
+    joined
+}
+
+/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is
+/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it.
+fn with_python_default_headers(
+    headers: Vec<(String, String)>,
+    defaults: &[(&str, &str)],
+) -> Vec<(String, String)> {
+    let missing: Vec<(String, String)> = defaults
+        .iter()
+        .filter(|(name, _)| !headers.iter().any(|(present, _)| present == name))
+        .map(|(name, value)| ((*name).to_string(), (*value).to_string()))
+        .collect();
+    headers.into_iter().chain(missing).collect()
+}
+
+/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier
+/// and sampling params before `additional_drop_params` and the environment check run.
+fn requested_optional_params(
+    request: MessagesRequest,
+    provider: MessagesProvider,
+    context: &MessagesTransformContext,
+) -> Result<MessagesRequest, Error> {
+    let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params(
+        request, context,
+    )?;
+    if provider.accepts_service_tier() {
+        return Ok(request);
+    }
+    Ok(MessagesRequest {
+        params: MessagesOptionalParams {
+            service_tier: None,
+            ..request.params
+        },
+        ..request
+    })
+}
+
 fn without_additional_drop_params(
     request: MessagesRequest,
     paths: &[String],
@@ -245,6 +306,7 @@ async fn call_provider(
     } = request;
     let provider_name = provider.as_str();
     log_request_body(provider_name, stream, &wire.body);
+    let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
     let response = send(
         http,
         Authenticated {
@@ -254,10 +316,11 @@ async fn call_provider(
         &wire.url,
         &wire.body,
         timeout,
+        stream,
     )
     .await?;
     if !response.status().is_success() {
-        return Err(provider_error(response).await);
+        return Err(provider_error(response, timeout).await);
     }
     let config = provider.config();
     if stream {
@@ -265,11 +328,24 @@ async fn call_provider(
             response,
             config.stream_decoder(),
             provider_name,
+            timeout,
         ));
     }
+    let head = ResponseHead {
+        status: response.status().as_u16(),
+        headers: litellm_http::request::header_pairs(response.headers()),
+    };
+    let status = head.status;
     let text = response.text().await.map_err(litellm_http::Error::from)?;
     log_response_body(&text);
-    context.response_received(&text).await?;
+    context
+        .raw_response_received(RawResponse::new(text.clone()).with_head(head))
+        .await?;
+    if serde_json::from_str::<serde::de::IgnoredAny>(&text).is_err() {
+        // Python raises `AnthropicError(status_code=<the response's>, message=<its text>)` for a
+        // body it cannot parse, and the public mapping classifies it by that status.
+        return Err(litellm_http::Error::upstream(status, text).into());
+    }
     decode_response(config, &identity.model, &text)
         .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
@@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error {
     ))
 }
 
+/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete
+/// response keeps the client's overall deadline; a stream is bounded until its head arrives and
+/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off.
 async fn send(
     http: &litellm_http::Client,
     authenticated: Authenticated,
     url: &str,
     body: &Value,
-    timeout: Option<Duration>,
+    timeout: Duration,
+    stream: bool,
 ) -> Result<reqwest::Response, Error> {
-    let request = outbound_request(
-        authenticated,
-        url,
-        body,
-        Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))),
-    )?;
-    Ok(litellm_inference::outbound::send(request, http).await?)
+    let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?;
+    if !stream {
+        return Ok(litellm_inference::outbound::send(request, http).await?);
+    }
+    tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http))
+        .await
+        .map_err(litellm_http::Error::read_timeout)?
+        .map_err(Error::from)
 }
 
-async fn provider_error(response: reqwest::Response) -> Error {
+/// The stream's body, each read bounded by `timeout` as httpx bounds each read.
+fn timed_chunks(
+    response: reqwest::Response,
+    timeout: Duration,
+) -> impl futures_util::Stream<Item = Result<Bytes, litellm_http::Error>> + Send + 'static {
+    futures_util::stream::try_unfold(response, move |mut response| async move {
+        let chunk = tokio::time::timeout(timeout, response.chunk())
+            .await
+            .map_err(litellm_http::Error::body_read_timeout)?
+            .map_err(litellm_http::Error::from)?;
+        Ok(chunk.map(|chunk| (chunk, response)))
+    })
+}
+
+/// The provider's error, its body read under `timeout`: a stream has no overall deadline once
+/// its head arrives, so an error body that stalls would otherwise hold the call forever.
+async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error {
     let status = response.status().as_u16();
-    match response.text().await {
-        Ok(text) => {
+    let headers = litellm_http::request::header_pairs(response.headers());
+    match tokio::time::timeout(timeout, response.text()).await {
+        Ok(Ok(text)) => {
             log_error_body(status, &text);
-            litellm_http::Error::upstream(status, text).into()
+            litellm_http::Error::upstream_with_headers(status, text, headers).into()
         }
-        Err(error) => litellm_http::Error::from(error).into(),
+        Ok(Err(error)) => litellm_http::Error::from(error).into(),
+        Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(),
     }
 }
 
@@ -329,22 +428,22 @@ fn streaming_response(
     response: reqwest::Response,
     decoder: Option<StreamDecoder>,
     provider: &'static str,
+    timeout: Duration,
 ) -> MessagesCallResponse {
     let headers = response
         .headers()
         .iter()
         .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string())))
         .collect();
+    let chunks = timed_chunks(response, timeout)
+        .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
     let chunks = match decoder {
-        None => futures_util::stream::try_unfold(response, move |mut response| async move {
-            let chunk = response.chunk().await.map_err(litellm_http::Error::from)?;
-            Ok(chunk.map(|chunk| {
-                log_chunk(provider, "provider_response", &chunk);
-                (chunk, response)
-            }))
-        })
-        .boxed(),
-        Some(decode) => decoded_chunks(response, decode, provider),
+        None => chunks.map_err(Error::from).boxed(),
+        Some(decode) => decoded_chunks(
+            chunks.map_err(std::io::Error::other).boxed(),
+            decode,
+            provider,
+        ),
     };
     MessagesCallResponse::Stream {
         head: super::route::MessagesStreamHead { headers },
@@ -353,15 +452,10 @@ fn streaming_response(
 }
 
 fn decoded_chunks(
-    response: reqwest::Response,
+    bytes: ByteStream,
     decode: StreamDecoder,
     provider: &'static str,
 ) -> BoxStream<'static, Result<Bytes, Error>> {
-    let bytes: ByteStream = response
-        .bytes_stream()
-        .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk))
-        .map_err(|error| std::io::Error::other(litellm_http::Error::from(error)))
-        .boxed();
     futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
         let Some(event) = events.try_next().await? else {
             return Ok(None);
@@ -964,9 +1058,12 @@ mod tests {
             .send()
             .await
             .unwrap();
-        let MessagesCallResponse::Stream { mut chunks, .. } =
-            streaming_response(response, Some(streaming::event_stream), "test")
-        else {
+        let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
+            response,
+            Some(streaming::event_stream),
+            "test",
+            Duration::from_secs(5),
+        ) else {
             panic!("a streaming response returns chunks");
         };
 

litellm-rust/crates/inference-messages/src/lib.rs

@@ -14,6 +14,7 @@ use std::sync::Arc;
 
 use crate::common_utils::messages_provider;
 
+pub use common_utils::merge_messages_header_sources;
 pub use litellm_inference::Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
 #[case::deepseek("deepseek")]
 #[case::minimax("minimax")]
 #[tokio::test]
-async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
+/// Python adds a default unless that exact lowercase name is present, then folds the casings
+/// into one header, so a capitalised caller header is sent beside the default.
+async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults(
+    call: MessagesCall,
+    #[case] provider: &str,
+) {
     let upstream = upstream([message_response()]).await;
 
     run_message(MessagesCall {
@@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
     .await;
 
     let request = only_request(&upstream).await;
-    assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]);
+    assert_eq!(
+        request.header_values("anthropic-version"),
+        ["2024-01-01, 2023-06-01"]
+    );
     assert_eq!(
         request.header_values("content-type"),
-        ["application/json; charset=utf-8"]
+        ["application/json; charset=utf-8, application/json"]
     );
 }
 

litellm-rust/crates/inference-messages/tests/messages/response.rs

@@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case]
 }
 
 #[rstest]
-#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))]
-#[case::not_a_message(json_response(json!({"unexpected": true})))]
 #[tokio::test]
-async fn an_unreadable_success_body_is_an_invalid_response(
-    call: MessagesCall,
-    #[case] response: ResponseTemplate,
-) {
-    let upstream = upstream([response]).await;
+async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) {
+    let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await;
+
+    let error = run(MessagesCall {
+        api_key: Some("sk".into()),
+        api_base: Some(upstream.uri()),
+        ..call
+    })
+    .await
+    .expect_err("an unparseable body fails");
+
+    assert_eq!(error.upstream_response(), Some((200, "not json")));
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) {
+    let upstream = upstream([json_response(json!({"unexpected": true}))]).await;
 
     let error = run(MessagesCall {
         api_key: Some("sk".into()),

litellm-rust/crates/inference-messages/tests/messages/stream.rs

@@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream(
         .unwrap();
 }
 
+/// Answers with an error head whose promised body never arrives.
+async fn stalling_error_upstream() -> (String, JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let base = format!("http://{}", listener.local_addr().unwrap());
+    let connection = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.unwrap();
+        let mut request = vec![0; 4096];
+        let _ = socket.read(&mut request).await;
+        socket
+            .write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n")
+            .await
+            .unwrap();
+        let _ = socket.read_to_end(&mut Vec::new()).await;
+    });
+    (base, connection)
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) {
+    let (base, connection) = stalling_error_upstream().await;
+    let host = RecordingStreamHost::new(
+        MessagesCall {
+            timeout: Some(Duration::from_millis(300)),
+            ..streaming(call, base)
+        },
+        usize::MAX,
+    );
+
+    let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host))
+        .await
+        .expect("the stalled error body gives up within the timeout")
+        .expect_err("a stalled error body fails the call");
+
+    assert!(error.dispatched(), "{error:?}");
+    assert_eq!(error.upstream_response(), None);
+    assert!(host.seen.into_inner().unwrap().is_empty());
+    tokio::time::timeout(Duration::from_secs(5), connection)
+        .await
+        .expect("timing out closes the upstream connection")
+        .unwrap();
+}
+
 #[rstest]
 #[tokio::test]
 async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) {

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -91,9 +91,7 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
     }
 
     fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-        let raw = RawResponse {
-            body: String::from_utf8_lossy(body).into_owned(),
-        };
+        let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned());
         if let Some(observers) = self.observers {
             observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference-responses/src/handler.rs

@@ -114,7 +114,7 @@ pub(super) async fn execute(
                 });
             }
             let body = response.text().await.map_err(litellm_http::Error::from)?;
-            let raw = RawResponse { body: body.clone() };
+            let raw = RawResponse::new(body.clone());
             if let Some(observers) = observers {
                 observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                     ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference/src/context.rs

@@ -42,9 +42,11 @@ impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
     }
 
     pub async fn response_received(&self, body: &str) -> Result<(), Error> {
-        let raw = RawResponse {
-            body: body.to_owned(),
-        };
+        self.raw_response_received(RawResponse::new(body.to_owned()))
+            .await
+    }
+
+    pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> {
         if let Some(observers) = &self.observers {
             observers.emit(CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference/src/error.rs

@@ -146,6 +146,34 @@ impl Error {
         }
     }
 
+    /// The provider's response headers on an upstream failure; empty for every other error.
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match self {
+            Self::Llm(error) => error.upstream_headers(),
+            Self::Http(error) => error.upstream_headers(),
+            _ => &[],
+        }
+    }
+
+    /// A read of a streamed body timed out after the response head arrived.
+    pub fn body_read_timeout(&self) -> bool {
+        match self {
+            Self::Llm(error) => error.is_body_read_timeout(),
+            Self::Http(error) => error.is_body_read_timeout(),
+            _ => false,
+        }
+    }
+
+    /// `Some(connect)` when the call timed out, where `connect` says the connection was never
+    /// established; `None` for every other error.
+    pub fn timeout(&self) -> Option<bool> {
+        match self {
+            Self::Llm(error) => error.is_timeout(),
+            Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+            _ => None,
+        }
+    }
+
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match self {
             Self::Llm(error) => error.upstream_response(),

litellm-rust/crates/llms-types/src/formats/messages/request.rs

@@ -264,7 +264,12 @@ pub enum ThinkingDisplay {
 #[macro_rules_attribute::apply(crate::wire_type)]
 #[derive(Default)]
 pub struct EnabledThinking {
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    /// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0.
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "crate::recognized::present"
+    )]
     pub budget_tokens: Option<Recognized<u64>>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub display: Option<Recognized<ThinkingDisplay>>,

litellm-rust/crates/llms-types/src/formats/messages/response.rs

@@ -10,14 +10,29 @@ pub struct MessagesResponse {
     pub content: Vec<Value>,
     pub stop_reason: Option<String>,
     pub stop_sequence: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    /// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps.
+    #[serde(
+        default,
+        deserialize_with = "present_value",
+        skip_serializing_if = "Option::is_none"
+    )]
     pub usage: Option<Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(
+        default,
+        deserialize_with = "present_value",
+        skip_serializing_if = "Option::is_none"
+    )]
     pub container: Option<Value>,
     #[serde(flatten)]
     pub extra: Map<String, Value>,
 }
 
+fn present_value<'de, D: serde::Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<Value>, D::Error> {
+    <Value as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -73,4 +88,21 @@ mod tests {
         assert_eq!(body.get("usage").cloned(), usage);
         assert_eq!(body.get("container").cloned(), container);
     }
+
+    #[test]
+    fn explicit_null_usage_and_container_survive_a_round_trip() {
+        let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
+            "content": [], "stop_reason": null, "stop_sequence": null,
+            "usage": null, "container": null});
+        let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
+        assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
+    }
+
+    #[test]
+    fn absent_usage_and_container_stay_absent() {
+        let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
+            "content": [], "stop_reason": null, "stop_sequence": null});
+        let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
+        assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
+    }
 }

litellm-rust/crates/llms-types/src/recognized.rs

@@ -7,6 +7,16 @@ pub enum Recognized<T> {
     Unrecognized(Value),
 }
 
+/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay
+/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`.
+pub fn present<'de, D, T>(deserializer: D) -> Result<Option<Recognized<T>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    Recognized<T>: serde::Deserialize<'de>,
+{
+    <Recognized<T> as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
+}
+
 impl<T> Recognized<T> {
     pub fn known(&self) -> Option<&T> {
         match self {

litellm-rust/crates/llms/src/anthropic/common_utils.rs

@@ -154,11 +154,42 @@ pub fn complete_anthropic_url(
 ) -> String {
     let api_base = resolve_anthropic_api_base(api_base, env_lookup);
 
-    let api_base = api_base.trim_end_matches('/');
-    if api_base.ends_with(MESSAGES_PATH) {
+    ensure_api_route(&api_base, MESSAGES_PATH)
+}
+
+/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds.
+fn url_path(url: &str) -> &str {
+    let before_fragment = url.split('#').next().unwrap_or_default();
+    let before_query = before_fragment.split('?').next().unwrap_or_default();
+    match before_query.split_once("://") {
+        Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]),
+        None => before_query,
+    }
+}
+
+/// Python's `_strip_trailing_slashes_keeping_the_scheme`.
+fn strip_trailing_slashes(api_base: &str) -> String {
+    let stripped = api_base.trim_end_matches('/');
+    if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') {
         return api_base.to_string();
     }
-    format!("{api_base}{MESSAGES_PATH}")
+    match stripped.split_once('?') {
+        Some((before, query)) if url_path(stripped).ends_with('/') => {
+            format!("{}?{query}", before.trim_end_matches('/'))
+        }
+        _ => stripped.to_string(),
+    }
+}
+
+/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as
+/// it is, query string included; anything else gets the route appended to the whole string.
+fn ensure_api_route(api_base: &str, route: &str) -> String {
+    let base = strip_trailing_slashes(api_base);
+    let route = route.trim_start_matches('/');
+    if url_path(&base).ends_with(&format!("/{route}")) {
+        return base;
+    }
+    format!("{}/{route}", strip_trailing_slashes(&base))
 }
 
 pub fn existing_betas(headers: &[(String, String)]) -> BetaSet {
@@ -1922,4 +1953,24 @@ mod tests {
             expected
         );
     }
+
+    #[rstest]
+    #[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")]
+    #[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")]
+    #[case::route_present(
+        "https://api.example.com/v1/messages/",
+        "https://api.example.com/v1/messages"
+    )]
+    #[case::route_and_query(
+        "https://api.example.com/v1/messages?beta=true",
+        "https://api.example.com/v1/messages?beta=true"
+    )]
+    #[case::query_without_route(
+        "https://api.example.com/?x=1",
+        "https://api.example.com?x=1/v1/messages"
+    )]
+    #[case::scheme_only("https://", "https:///v1/messages")]
+    fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) {
+        assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url);
+    }
 }

litellm-rust/crates/llms/src/anthropic/messages/thinking.rs

@@ -2,8 +2,8 @@ use litellm_llms_types::{
     formats::{
         chat_completions::ReasoningEffort,
         messages::{
-            EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig,
-            ThinkingDisplay,
+            AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig,
+            ThinkingConfig, ThinkingDisplay,
         },
     },
     recognized::Recognized,
@@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti
 
 fn effort_for_budget(
     budgets: &ThinkingBudgets,
-    budget_tokens: u64,
+    budget_tokens: i64,
     capabilities: &MessagesModelCapabilities,
 ) -> EffortLevel {
-    if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh {
+    let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier);
+    if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh {
         return EffortLevel::Xhigh;
     }
-    if budget_tokens >= budgets.high {
+    if reaches(budgets.high) {
         return EffortLevel::High;
     }
-    if budget_tokens >= budgets.medium {
+    if reaches(budgets.medium) {
         return EffortLevel::Medium;
     }
     EffortLevel::Low
@@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) -
     }
 }
 
+/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string
+/// as a budget, and nothing else.
+fn numeric_budget_tokens(budget: &Recognized<u64>) -> Option<i64> {
+    let value = match budget {
+        Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)),
+        Recognized::Unrecognized(value) => value,
+    };
+    match value {
+        Value::Number(number) => number
+            .as_i64()
+            .or_else(|| number.as_f64().map(|float| float.trunc() as i64)),
+        Value::String(text) => python_int(text),
+        Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None,
+    }
+}
+
+/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits.
+fn python_int(text: &str) -> Option<i64> {
+    let trimmed = text.trim();
+    let digits = trimmed.trim_start_matches(['+', '-']);
+    let well_formed = trimmed.len() - digits.len() <= 1
+        && !digits.is_empty()
+        && !digits.starts_with('_')
+        && !digits.ends_with('_')
+        && !digits.contains("__")
+        && digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_');
+    if !well_formed {
+        return None;
+    }
+    let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect();
+    cleaned.parse::<i64>().ok().or_else(|| {
+        Some(if cleaned.starts_with('-') {
+            i64::MIN
+        } else {
+            i64::MAX
+        })
+    })
+}
+
+fn python_type_name(value: &Value) -> &'static str {
+    match value {
+        Value::Null => "NoneType",
+        Value::Bool(_) => "bool",
+        Value::Number(number) if number.is_f64() => "float",
+        Value::Number(_) => "int",
+        Value::String(_) => "str",
+        Value::Array(_) => "list",
+        Value::Object(_) => "dict",
+    }
+}
+
+fn kept_display(
+    display: Option<&Recognized<ThinkingDisplay>>,
+    context: &ThinkingContext,
+) -> Option<Recognized<ThinkingDisplay>> {
+    match display? {
+        Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => {
+            display.cloned()
+        }
+        Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => {
+            display.cloned()
+        }
+        Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None,
+    }
+}
+
+/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive
+/// keys always, and any other key unless `drop_params` is set.
+fn carried_thinking_keys(
+    extra: &serde_json::Map<String, Value>,
+    context: &ThinkingContext,
+) -> serde_json::Map<String, Value> {
+    extra
+        .iter()
+        .filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params)
+        .map(|(key, value)| (key.clone(), value.clone()))
+        .collect()
+}
+
+/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`.
 fn translate_legacy_thinking_for_adaptive_model(
     request: MessagesRequest,
     context: &ThinkingContext,
-) -> MessagesRequest {
+) -> Result<MessagesRequest, Error> {
     let capabilities = &context.capabilities;
     if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking {
-        return request;
+        return Ok(request);
     }
     let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else {
-        return request;
+        return Ok(request);
     };
-    let budget = enabled
-        .budget_tokens
-        .as_ref()
-        .and_then(Recognized::known)
-        .copied()
-        .unwrap_or(0);
-    let level = effort_for_budget(&context.budgets, budget, capabilities);
-    MessagesRequest {
+    let budget = match &enabled.budget_tokens {
+        None => Some(0),
+        Some(budget) => numeric_budget_tokens(budget),
+    };
+    if budget.is_none() && !context.drop_params {
+        let actual = match &enabled.budget_tokens {
+            Some(Recognized::Unrecognized(value)) => python_type_name(value),
+            Some(Recognized::Known(_)) | None => "int",
+        };
+        return Err(Error::InvalidRequest(
+            crate::ErrorDetail::NonNumericThinkingBudget { actual },
+        ));
+    }
+    let thinking = ThinkingConfig::Adaptive(AdaptiveThinking {
+        display: kept_display(enabled.display.as_ref(), context),
+        extra: carried_thinking_keys(&enabled.extra, context),
+    });
+    let output_config = match budget {
+        Some(budget) => with_default_effort(
+            request.params.output_config,
+            effort_for_budget(&context.budgets, budget, capabilities),
+        ),
+        None => request.params.output_config,
+    };
+    Ok(MessagesRequest {
         params: MessagesOptionalParams {
-            thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))),
-            output_config: with_default_effort(request.params.output_config, level),
+            thinking: Some(Recognized::Known(thinking)),
+            output_config,
             ..request.params
         },
         ..request
-    }
+    })
 }
 
 fn translate_adaptive_effort_for_non_adaptive_model(
@@ -337,7 +435,7 @@ pub fn translate_thinking(
 ) -> Result<MessagesRequest, Error> {
     let request = translate_reasoning_effort(request, context)?;
     let request = drop_disabled_thinking(request, context);
-    let request = translate_legacy_thinking_for_adaptive_model(request, context);
+    let request = translate_legacy_thinking_for_adaptive_model(request, context)?;
     let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?;
     Ok(drop_incompatible_temperature_for_thinking(request, context))
 }
@@ -363,6 +461,7 @@ mod tests {
         ThinkingContext {
             capabilities,
             budgets: ThinkingBudgets::default(),
+            ..ThinkingContext::default()
         }
     }
 
@@ -1239,10 +1338,102 @@ mod tests {
         let context = ThinkingContext {
             capabilities,
             budgets: overridden_budgets(overrides),
+            ..ThinkingContext::default()
         };
         assert_eq!(
             translate_thinking(request(input), &context),
             Ok(request(expected))
         );
     }
+
+    use serde_json::json;
+
+    fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext {
+        ThinkingContext {
+            drop_params,
+            keeps_display_updates,
+            ..context(opus_4_7())
+        }
+    }
+
+    #[rstest]
+    #[case::float_budget(json!(8192.0), "xhigh")]
+    #[case::float_below_a_tier(json!(1023.9), "low")]
+    #[case::string_budget(json!(" 16_000 "), "xhigh")]
+    #[case::negative_budget(json!(-5), "low")]
+    fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) {
+        let translated = translate_thinking(
+            request(
+                json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}),
+            ),
+            &legacy_context(false, false),
+        );
+        assert_eq!(
+            translated,
+            Ok(request(json!({
+                "max_tokens": 64000,
+                "thinking": {"type": "adaptive"},
+                "output_config": {"effort": effort},
+            })))
+        );
+    }
+
+    #[rstest]
+    #[case::text(json!("lots"), "str")]
+    #[case::list(json!([1]), "list")]
+    #[case::null(Value::Null, "NoneType")]
+    #[case::boolean(json!(true), "bool")]
+    fn a_non_numeric_legacy_budget_is_rejected_unless_dropped(
+        #[case] budget: Value,
+        #[case] actual: &'static str,
+    ) {
+        let fields =
+            json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}});
+        assert_eq!(
+            translate_thinking(request(fields.clone()), &legacy_context(false, false)),
+            Err(Error::InvalidRequest(
+                crate::ErrorDetail::NonNumericThinkingBudget { actual }
+            ))
+        );
+        assert_eq!(
+            translate_thinking(request(fields), &legacy_context(true, false)),
+            Ok(request(
+                json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}})
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::summarized("summarized", false, Some("summarized"))]
+    #[case::updates_kept("updates", true, Some("updates"))]
+    #[case::updates_dropped("updates", false, None)]
+    #[case::unknown("verbose", true, None)]
+    fn legacy_display_survives_only_where_python_keeps_it(
+        #[case] display: &str,
+        #[case] keeps_display_updates: bool,
+        #[case] kept: Option<&str>,
+    ) {
+        let translated = translate_thinking(
+            request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})),
+            &legacy_context(false, keeps_display_updates),
+        )
+        .unwrap();
+        let thinking = serde_json::to_value(translated.params.thinking).unwrap();
+        assert_eq!(thinking.get("display").and_then(Value::as_str), kept);
+    }
+
+    #[rstest]
+    #[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))]
+    #[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))]
+    fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) {
+        let translated = translate_thinking(
+            request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})),
+            &legacy_context(drop_params, false),
+        )
+        .unwrap();
+        assert_eq!(
+            serde_json::to_value(translated.params.thinking).unwrap(),
+            expected
+        );
+    }
 }

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         request: MessagesRequest,
         context: &MessagesTransformContext,
     ) -> Result<MessagesRequest, Error> {
-        transform_messages_request(request, context)
+        transform_messages_request(request, &context.keeping_display_updates())
     }
 
     fn secret_names(&self) -> &'static [&'static str] {
@@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
             .then_some(AnthropicBeta::PerTurnControl20260701),
         uses_thinking_display_updates(params)
             .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
+        uses_thinking_block_binding(params)
+            .then_some(AnthropicBeta::ThinkingBindingControls20260801),
         messages_carry_tool_changes(&request.messages)
             .then_some(AnthropicBeta::MidConversationToolChanges20260701),
         has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
@@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
     display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
 }
 
+/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`.
+fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool {
+    match params.thinking.as_ref() {
+        Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Unrecognized(Value::Object(thinking))) => {
+            thinking.contains_key("block_binding")
+        }
+        Some(Recognized::Unrecognized(_)) | None => false,
+    }
+}
+
 fn is_tool_reference_change(block: &ContentBlock) -> bool {
     matches!(
         &block.block_type,
@@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error
     })
 }
 
-fn drop_unsupported_params(
+/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`.
+pub fn drop_unsupported_params(
     request: MessagesRequest,
     context: &MessagesTransformContext,
 ) -> Result<MessagesRequest, Error> {
@@ -838,6 +860,8 @@ mod tests {
                             low: 2000,
                             ..ThinkingBudgets::default()
                         },
+                        drop_params: true,
+                        keeps_display_updates: false,
                     },
                     drop_params: true,
                 }

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
                     .capabilities
                     .supports_mid_conversation_system,
             )),
-            context,
+            &context.keeping_display_updates(),
         )
     }
 

litellm-rust/crates/llms/src/base_llm/messages/context.rs

@@ -110,6 +110,11 @@ impl ThinkingBudgets {
 pub struct ThinkingContext {
     pub capabilities: MessagesModelCapabilities,
     pub budgets: ThinkingBudgets,
+    /// `drop_params` as Python's thinking translation reads it.
+    pub drop_params: bool,
+    /// The provider accepts `display: "updates"` on thinking translated from the legacy shape
+    /// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`).
+    pub keeps_display_updates: bool,
 }
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub struct MessagesTransformContext {
@@ -130,8 +135,22 @@ impl MessagesTransformContext {
             thinking: ThinkingContext {
                 capabilities,
                 budgets: ThinkingBudgets::from_lookup(env),
+                drop_params,
+                keeps_display_updates: false,
             },
             drop_params,
         }
     }
+
+    /// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`.
+    #[must_use]
+    pub fn keeping_display_updates(&self) -> Self {
+        Self {
+            thinking: ThinkingContext {
+                keeps_display_updates: true,
+                ..self.thinking
+            },
+            ..*self
+        }
+    }
 }

litellm-rust/crates/llms/src/error.rs

@@ -92,6 +92,24 @@ impl Error {
         }
     }
 
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match self {
+            Self::Http(error) => error.upstream_headers(),
+            _ => &[],
+        }
+    }
+
+    pub fn is_body_read_timeout(&self) -> bool {
+        matches!(self, Self::Http(error) if error.is_body_read_timeout())
+    }
+
+    pub fn is_timeout(&self) -> Option<bool> {
+        match self {
+            Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+            _ => None,
+        }
+    }
+
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match self {
             Self::Http(error) => error.upstream_response(),
@@ -158,6 +176,10 @@ pub enum ErrorDetail {
         value: String,
         hint: String,
     },
+    #[error(
+        "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy"
+    )]
+    NonNumericThinkingBudget { actual: &'static str },
     #[error(
         "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params"
     )]

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         request: MessagesRequest,
         context: &MessagesTransformContext,
     ) -> Result<MessagesRequest, Error> {
-        let request = transform_messages_request(request, context)?;
+        let request = transform_messages_request(request, &context.keeping_display_updates())?;
         let request = strip_cache_control_scope(normalize_system_role_messages(
             request,
             context

litellm-rust/crates/model-catalog/src/vocabulary.rs

@@ -93,6 +93,12 @@ pub enum Endpoint {
     Realtime,
     #[serde(rename = "/v1/realtime/transcription_sessions")]
     RealtimeTranscriptionSessions,
+    #[serde(rename = "/v1/realtime/translations")]
+    RealtimeTranslations,
+    #[serde(rename = "/v1/realtime/translations/calls")]
+    RealtimeTranslationCalls,
+    #[serde(rename = "/v1/realtime/translations/client_secrets")]
+    RealtimeTranslationClientSecrets,
     #[serde(rename = "/v1/videos")]
     Videos,
     #[serde(rename = "/v1/ocr")]

litellm-rust/crates/python-bridge/src/errors.rs

@@ -23,6 +23,17 @@ pyo3::create_exception!(
 );
 
 const MESSAGES_REQUEST_ERROR: &str = "messages_request_error";
+/// The rejection is a parameter the model does not support, which Python raises as
+/// `UnsupportedParamsError`.
+const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param";
+/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or
+/// `Body` for a streamed body read that timed out after the head arrived.
+const MESSAGES_TIMEOUT: &str = "messages_timeout";
+/// No credential reached the provider, which Python raises as `AuthenticationError`.
+const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential";
+/// Python refuses the request only once its logger knows the resolved model and provider (the
+/// provider config's request transform), so the failure logs them.
+const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection";
 
 pub(crate) fn upstream_error(
     py: Python<'_>,
@@ -78,17 +89,57 @@ pub(crate) fn route_failure(
         return Ok(raised);
     }
     if let Some((status, body)) = error.upstream_response() {
-        return public(upstream_error(py, status, body, &HeaderMap::new())?);
+        let headers = error
+            .upstream_headers()
+            .iter()
+            .filter_map(|(name, value)| {
+                Some((
+                    reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?,
+                    reqwest::header::HeaderValue::from_str(value).ok()?,
+                ))
+            })
+            .collect::<HeaderMap>();
+        return public(upstream_error(py, status, body, &headers)?);
     }
     let native = match (marker, error.rejected_request()) {
         (RequestMarker::Messages, Some(rejection)) => {
             let native = PyValueError::new_err(Report::new(rejection).to_string());
-            native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?;
+            let value = native.value(py);
+            value.setattr(MESSAGES_REQUEST_ERROR, true)?;
+            if matches!(
+                rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
+                Some(
+                    litellm_llms::ErrorDetail::UnsupportedParameter { .. }
+                        | litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }
+                )
+            ) {
+                value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?;
+            }
+            if matches!(
+                rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
+                Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. })
+            ) {
+                value.setattr(MESSAGES_LATE_REJECTION, true)?;
+            }
             native
         }
-        (RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => {
-            by_fault(error.is_request(), &error)
+        (RequestMarker::Messages, None) => {
+            let native = by_fault(error.is_request(), &error);
+            let value = native.value(py);
+            if let Some(connect) = error.timeout() {
+                let phase = match (connect, error.body_read_timeout()) {
+                    (true, _) => "Connection",
+                    (false, true) => "Body",
+                    (false, false) => "Read",
+                };
+                value.setattr(MESSAGES_TIMEOUT, phase)?;
+            }
+            if error.is_missing_credential() {
+                value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?;
+            }
+            native
         }
+        (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error),
     };
     public(native)
 }

litellm-rust/crates/python-bridge/src/routes/messages/host.rs

@@ -34,11 +34,9 @@ fn merge_headers(
     forwarded: Option<Map<String, Value>>,
     extra_headers: Option<Map<String, Value>>,
 ) -> Option<Map<String, Value>> {
-    let merged: Map<String, Value> = forwarded
-        .into_iter()
-        .flatten()
-        .chain(extra_headers.into_iter().flatten())
-        .collect();
+    let merged = litellm_inference_messages::merge_messages_header_sources(
+        forwarded.into_iter().chain(extra_headers),
+    );
     (!merged.is_empty()).then_some(merged)
 }
 
@@ -107,10 +105,19 @@ impl MessagesPythonHost {
             .map(|value| python_timeout_seconds(py, value.unbind()))
             .transpose()?
             .flatten();
-        let custom_llm_provider = string("custom_llm_provider")?;
-        let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
         let api_key = string("api_key")?;
         let api_base = string("api_base")?;
+        // Python's handler resolves a bare model name (`claude-sonnet-4-5`) through
+        // `get_llm_provider`'s model lists, which the native prefix lookup does not carry.
+        let custom_llm_provider = match string("custom_llm_provider")? {
+            Some(provider) => Some(provider),
+            None => py
+                .import(ROUTE_HOST_MODULE)?
+                .getattr("inferred_provider")?
+                .call1((&model, api_base.as_deref()))?
+                .extract()?,
+        };
+        let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
         let extra_headers = self.merged_headers(py, arguments)?;
         let provider_specific_header = self.provider_specific_header(py, arguments)?;
         let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;

litellm-rust/crates/python-bridge/src/routes/messages/mod.rs

@@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
     )
 }
 
+/// Python's sync Messages handler raises for every native provider before it sends anything
+/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies
+/// that until both sides serve sync calls.
+const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls";
+
 #[pyfunction]
-pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
-    run_messages(py, call, false)
+#[pyo3(signature = (call))]
+pub(crate) fn messages(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
+    drop(call);
+    Err(pyo3::exceptions::PyValueError::new_err(
+        SYNC_NOT_IMPLEMENTED,
+    ))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -8,7 +8,7 @@ _apply_process_defaults()
 import threading
 import os
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from typing import (
     Any,
     Callable,
@@ -1562,7 +1562,9 @@ from .realtime_api.main import (
     _arealtime,
     acreate_realtime_client_secret,
     acreate_realtime_transcription_session,
+    acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
     arealtime_calls,
+    arealtime_translation_calls as arealtime_translation_calls,
 )
 from .responses.main import _aresponses_websocket
 
@@ -1754,6 +1756,9 @@ if TYPE_CHECKING:
     from .llms.typesafe.decisions.transformation import (
         TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
     )
+    from .llms.parallel_ai.decisions.transformation import (
+        ParallelAIDecisionsConfig as ParallelAIDecisionsConfig,
+    )
     from .llms.openrouter.decisions.transformation import (
         OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
     )
@@ -1808,6 +1813,9 @@ if TYPE_CHECKING:
     from .llms.vertex_ai.rerank.transformation import (
         VertexAIRerankConfig as VertexAIRerankConfig,
     )
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.fireworks_ai.rerank.transformation import (
         FireworksAIRerankConfig as FireworksAIRerankConfig,
     )
@@ -1853,9 +1861,6 @@ if TYPE_CHECKING:
         BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
-    from .llms.together_ai.chat.transformation import (
-        TogetherAIChatConfig as TogetherAIChatConfig,
-    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -2014,6 +2019,9 @@ if TYPE_CHECKING:
     from .llms.xai.responses.transformation import (
         XAIResponsesAPIConfig as XAIResponsesAPIConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.litellm_proxy.responses.transformation import (
         LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
     )
@@ -2041,9 +2049,6 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
-    from .llms.vertex_ai.interactions.transformation import (
-        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
-    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,

litellm/_internal_context.py

@@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
         internal_parent_request_id.reset(parent_token)
 
 
+@contextmanager
+def billed_separately() -> Generator[None]:
+    """A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
+    token: Final = is_internal_call.set(False)
+    parent_token: Final = internal_parent_request_id.set(None)
+    try:
+        yield
+    finally:
+        is_internal_call.reset(token)
+        internal_parent_request_id.reset(parent_token)
+
+
 @contextmanager
 def post_response_phase() -> Generator[None]:
     """Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

litellm/_lazy_imports_registry.py

@@ -163,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
     "HostedVLLMRerankConfig",
     "PerplexityDecisionsConfig",
     "TypeSafeDecisionsConfig",
+    "ParallelAIDecisionsConfig",
     "NaceDecisionsConfig",
     "OpenRouterDecisionsConfig",
     "FireworksAIDecisionsConfig",
@@ -740,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
     "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
     "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
+    "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
     "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
     "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
     "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),

litellm/anthropic_beta_headers_manager.py

@@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import cache
 from importlib.resources import files
 from typing import Final, Literal, TypeAlias
+from urllib.parse import urlparse
 
 import httpx
 from pydantic import TypeAdapter
@@ -48,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 _DECODED_JSON: Final = TypeAdapter(object)
 
 BetaPlacement: TypeAlias = Literal["header", "body"]
+UnlistedBetas: TypeAlias = Literal["drop", "forward"]
 _HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
 
 
@@ -280,12 +282,13 @@ def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
     placement: BetaPlacement = "header",
+    unlisted: UnlistedBetas = "drop",
 ) -> list[str]:
     """
     Filter and transform beta headers based on provider's mapping configuration.
 
     This function:
-    1. Only allows headers that are present in the provider's mapping keys
+    1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
     2. Filters out headers with null values (unsupported)
     3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
 
@@ -293,6 +296,7 @@ def filter_and_transform_beta_headers(
         beta_headers: List of Anthropic beta header values
         provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
         placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
+        unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
 
     Returns:
         List of filtered and transformed beta headers for the provider
@@ -307,6 +311,10 @@ def filter_and_transform_beta_headers(
     for header in beta_headers:
         header = header.strip()
 
+        if header not in provider_mapping and unlisted == "forward":
+            filtered_headers.add(header)
+            continue
+
         # Check if header is in the mapping
         if header not in provider_mapping:
             verbose_logger.debug(
@@ -388,9 +396,38 @@ def get_provider_beta_header(
     return provider_mapping[anthropic_beta_header]
 
 
+FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
+    "services.ai.azure.com",
+    "openai.azure.com",
+    "cognitiveservices.azure.com",
+)
+
+
+def is_foundry_api_base(api_base: str | None) -> bool:
+    """Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft."""
+    if not api_base:
+        return False
+    host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or ""
+    return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES)
+
+
+def resolve_beta_provider(provider: str, api_base: str | None = None) -> str:
+    """
+    Pick the beta allowlist row for one deployment, not one provider string.
+
+    An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and
+    gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com.
+    """
+    if provider == "anthropic" and is_foundry_api_base(api_base):
+        return "azure_ai"
+    return provider
+
+
 def update_headers_with_filtered_beta(
     headers: dict,
     provider: str,
+    api_base: str | None = None,
+    unlisted: UnlistedBetas = "drop",
 ) -> dict:
     """
     Update headers dict by filtering and transforming anthropic-beta header values.
@@ -399,6 +436,8 @@ def update_headers_with_filtered_beta(
     Args:
         headers: Request headers dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
+        unlisted: Whether a beta missing from the row is dropped or forwarded as sent
 
     Returns:
         Updated headers dict
@@ -413,7 +452,8 @@ def update_headers_with_filtered_beta(
     # Filter and transform based on provider
     filtered_beta_values: Final = filter_and_transform_beta_headers(
         beta_headers=beta_values,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
+        unlisted=unlisted,
     )
 
     # Update or remove the header
@@ -430,6 +470,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
+    api_base: str | None = None,
 ) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
@@ -439,11 +480,12 @@ def update_request_with_filtered_beta(
         headers: Request headers dict (will be modified in place)
         request_data: Request body dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
 
     Returns:
         Tuple of (updated headers, updated request_data)
     """
-    headers = update_headers_with_filtered_beta(headers=headers, provider=provider)
+    headers = update_headers_with_filtered_beta(headers=headers, provider=provider, api_base=api_base)
 
     existing_body_betas: Final = request_data.get("anthropic_beta")
     if not existing_body_betas:
@@ -451,7 +493,7 @@ def update_request_with_filtered_beta(
 
     filtered_body_betas: Final = filter_and_transform_beta_headers(
         beta_headers=existing_body_betas,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
         placement="body",
     )
 

litellm/anthropic_interface/exceptions/exception_mapping_utils.py

@@ -16,7 +16,7 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE
 _PARSED_ERROR_BODY: Final = TypeAdapter(object)
 
 # HTTP status code -> Anthropic error type
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
     400: "invalid_request_error",
     401: "authentication_error",
@@ -46,6 +46,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Create an Anthropic-formatted error response dict.
@@ -57,12 +58,10 @@ class AnthropicExceptionMapping:
             "request_id": "req_..."
         }
         """
-        error_type: Final = AnthropicExceptionMapping.get_error_type(status_code)
-
         response: Final[AnthropicErrorResponse] = {
             "type": "error",
             "error": {
-                "type": error_type,
+                "type": error_type or AnthropicExceptionMapping.get_error_type(status_code),
                 "message": message,
             },
         }
@@ -131,6 +130,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         raw_message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Transform an error message to Anthropic format.
@@ -144,6 +144,7 @@ class AnthropicExceptionMapping:
             status_code: HTTP status code
             raw_message: Raw error message (may be JSON string or plain text)
             request_id: Optional request ID to include
+            error_type: Anthropic error type to use instead of the one the status maps to
 
         Returns:
             AnthropicErrorResponse dict
@@ -169,6 +170,7 @@ class AnthropicExceptionMapping:
             status_code=status_code,
             message=message,
             request_id=request_id,
+            error_type=error_type,
         )
 
 

litellm/anthropic_interface/exceptions/exceptions.py

@@ -6,15 +6,18 @@ from typing import Literal
 from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 # Known Anthropic error types
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 AnthropicErrorType = Literal[
     "invalid_request_error",
     "authentication_error",
+    "billing_error",
     "permission_error",
     "not_found_error",
+    "conflict_error",
     "request_too_large",
     "rate_limit_error",
     "api_error",
+    "timeout_error",
     "overloaded_error",
 ]
 
@@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict):
     type: AnthropicErrorType
     message: str
     provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
+    litellm_error_type: NotRequired[ReadOnly[str]]
     litellm_call_id: NotRequired[ReadOnly[str]]
 
 

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import (
     ResponseCustomToolCallOutputParam,
     ResponseCustomToolCallParam,
 )
+from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
 from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
 from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
 from openai.types.responses.tool_param import FunctionToolParam
@@ -33,7 +34,11 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
     replay_items,
     requested_phase,
 )
-from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
+from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
+from litellm.litellm_core_utils.allowed_tools_tool_choice import (
+    parse_chat_allowed_tools_tool_choice,
+    responses_allowed_tools_tool_choice,
+)
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
@@ -783,13 +788,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def _normalize_tool_choice_for_responses_api(
         self, tool_choice: _ToolChoiceT
-    ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
+    ) -> (
+        _ToolChoiceT
+        | ToolChoiceFunctionParam
+        | ToolChoiceCustomParam
+        | ToolChoiceAllowedParam
+        | Literal["auto", "none", "required"]
+    ):
         """Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
         if not isinstance(tool_choice, dict):
             return tool_choice
         choice_type: Final = tool_choice.get("type")
         if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
             return choice_type
+        if choice_type == "allowed_tools":
+            parsed: Final = parse_chat_allowed_tools_tool_choice(
+                cast(  # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
+                    object, tool_choice
+                )
+            )
+            flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None
+            if flat is not None:
+                return flat
         if choice_type not in ("function", "custom"):
             return tool_choice
         if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
@@ -1146,10 +1166,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         optional_params = self._extract_extra_body_params(optional_params)
         provider: Final = _served_provider(litellm_params)
         base_model: Final = litellm_params.get("base_model")
-        supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or (
-            isinstance(base_model, str)
-            and bool(base_model)
-            and supports_openai_prompt_cache_breakpoint(base_model, provider)
+        model_info: Final = routed_model_info(litellm_params.get("model_info"))
+        named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,)
+        supports_prompt_cache_breakpoint: Final = any(
+            supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
         )
         converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
             messages,

litellm/constants.py

@@ -451,6 +451,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
     {"response.completed", "response.incomplete", "response.failed"}
 )
 REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
+REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally"
 
 # SSL/TLS cipher configuration for faster handshakes
 # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
@@ -971,7 +972,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
 
 OPENAI_TRANSCRIPTION_PARAMS: Final = [
     "language",
+    "languages",
+    "keywords",
     "response_format",
+    "stream",
     "timestamp_granularities",
 ]
 

litellm/cost_calculator.py

@@ -23,9 +23,14 @@ from litellm.constants import (
     OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
     REALTIME_METERED_USAGE_KEY,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
+from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
-from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
+from litellm.litellm_core_utils.decisions_pricing import (
+    decisions_pricing_fields,
+    decisions_request_price,
+    is_decisions_call_type,
+)
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
@@ -320,12 +325,41 @@ def _get_additional_costs(
     return None
 
 
-def _transcription_usage_has_token_details(
+_TranscriptionRateKey: TypeAlias = Literal[
+    "input_cost_per_second",
+    "output_cost_per_second",
+    "input_cost_per_audio_token",
+    "output_cost_per_audio_token",
+    "input_cost_per_token",
+    "output_cost_per_token",
+]
+_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_audio_token",
+    "output_cost_per_audio_token",
+)
+_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
+    "input_cost_per_second",
+    "output_cost_per_second",
+)
+
+
+def _transcription_uses_token_pricing(
+    model: str,
+    custom_llm_provider: str | None,
     usage_block: Usage | None,
 ) -> bool:
     if usage_block is None:
         return False
 
+    model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
+    if (
+        _declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
+        and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
+    ):
+        return False
+
     prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
     completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
     prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
@@ -339,13 +373,38 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
-def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
-    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
+    if pricing_model is None or custom_llm_provider is None:
         return False
+    transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
+    transcription_metadata: Final = (
+        as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
+    )
+    return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
+
+
+def _transcription_duration_for_cost(
+    completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
+) -> float:
+    hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
+    measured_duration: Final = normalized_audio_duration_seconds(
+        hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
+    )
+    if measured_duration is not None and measured_duration > 0:
+        return measured_duration
+    provider_duration: Final = _get_transcription_usage_duration(completion_response)
+    if (
+        provider_duration is not None
+        and provider_duration > 0
+        and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
+    ):
+        return provider_duration
+    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+        return 0.0
     transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
         model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
     )
-    return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
+    return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
 
 
 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@@ -605,6 +664,9 @@ def cost_per_token(
         decisions_overrides: Final = decisions_pricing_fields(
             litellm.model_cost.get(decisions_model_info["key"], {})  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # model cost map is dynamically loaded
         )
+        request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides)
+        if request_price is not None:
+            return request_price, 0.0
         if decisions_overrides:
             return generic_cost_per_token(
                 model=model,
@@ -694,7 +756,7 @@ def cost_per_token(
             vertex_location=vertex_location,
         )
     elif call_type == "atranscription" or call_type == "transcription":
-        if _transcription_usage_has_token_details(usage_block):
+        if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
             return generic_cost_per_token(
                 model=model,
                 usage=usage_block,
@@ -1276,6 +1338,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
     )
 
 
+def _get_transcription_usage_duration(completion_response: object) -> float | None:
+    response_fields: Final = as_str_mapping(completion_response)
+    usage_object: Final = (
+        response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
+    )
+    usage_fields: Final = as_str_mapping(usage_object)
+    usage_type: Final = (
+        usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
+    )
+    if usage_type != "duration":
+        return None
+    seconds: Final = (
+        usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
+    )
+    return normalized_audio_duration_seconds(seconds)
+
+
 def _extraction_usage(usage: ExtractionUsage) -> Usage:
     return Usage(
         prompt_tokens=usage.input_tokens,
@@ -2001,20 +2080,8 @@ def completion_cost(
                         prompt,
                     )
                 elif call_type in _TRANSCRIPTION_CALL_TYPES:
-                    # Check _hidden_params first (duration stored there to
-                    # avoid polluting the response body), then fall back to
-                    # the response attribute (for verbose_json responses that
-                    # naturally include duration from the provider).
-                    _hidden = getattr(completion_response, "_hidden_params", {}) or {}
-                    audio_transcription_file_duration = _hidden.get(
-                        "audio_transcription_duration",
-                        getattr(completion_response, "duration", 0.0),
-                    ) or (
-                        completion_response.usage.seconds
-                        if isinstance(completion_response, TranscriptionResponse)
-                        and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
-                        and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
-                        else 0.0
+                    audio_transcription_file_duration = _transcription_duration_for_cost(
+                        completion_response, model, requested_model, custom_llm_provider
                     )
                 elif call_type in _RERANK_CALL_TYPES:
                     if completion_response is not None and isinstance(completion_response, RerankResponse):
@@ -2124,6 +2191,7 @@ def completion_cost(
                         litellm_logging_obj=litellm_logging_obj,
                         custom_pricing_model=selected_model if custom_pricing else None,
                         base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
+                        deployment_models=(base_model, requested_model),
                     )
                 elif call_type == _MCP_CALL_TYPE:
                     from litellm.proxy.mcp_server.cost_calculator import (
@@ -3402,8 +3470,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
     def create_logging_realtime_object(
         usage: Usage, results: OpenAIRealtimeStreamList
     ) -> LiteLLMRealtimeStreamLoggingObject:
+        translated_input_seconds, _ = _translation_session_seconds(results)
         return LiteLLMRealtimeStreamLoggingObject(
-            usage=usage,
+            usage=usage.model_copy(update={"seconds": translated_input_seconds})
+            if translated_input_seconds > 0
+            else usage,
             results=results,
         )
 
@@ -3488,6 +3559,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
+_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
 
 
 class _LiveResponsePayload(BaseModel):
@@ -3710,6 +3782,7 @@ def handle_realtime_stream_cost_calculation(
     litellm_logging_obj: LitellmLoggingObject | None = None,
     custom_pricing_model: str | None = None,
     base_pricing_model: str | None = None,
+    deployment_models: Sequence[str | None] = (),
 ) -> float:
     """
     Handles the cost calculation for realtime stream responses.
@@ -3754,6 +3827,13 @@ def handle_realtime_stream_cost_calculation(
         if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
         else 0.0
     )
+    translation_cost: Final = handle_realtime_translation_cost_calculation(
+        results=results,
+        custom_llm_provider=custom_llm_provider,
+        litellm_model_name=litellm_model_name,
+        deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
+        session_model=received_model,
+    )
     live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
     live_session_started: Final = any(result["type"] == "session.started" for result in results)
     live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
@@ -3810,12 +3890,19 @@ def handle_realtime_stream_cost_calculation(
     input_cost: Final = input_cost_per_token + live_response_input_cost
     output_cost: Final = output_cost_per_token + live_response_output_cost
     total_cost: Final = (
-        input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
+        input_cost
+        + output_cost
+        + transcription_cost
+        + translation_cost
+        + live_voice_cost
+        + audio_duration_cost
+        + text_message_cost
     )
     additional_costs: Final = tuple(
         (cost_name, cost)
         for cost_name, cost in (
             ("transcription_cost", transcription_cost),
+            ("translation_cost", translation_cost),
             ("live_voice_cost", live_voice_cost),
             ("audio_duration_cost", audio_duration_cost),
             ("text_message_cost", text_message_cost),
@@ -3836,6 +3923,55 @@ def handle_realtime_stream_cost_calculation(
     return total_cost
 
 
+def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
+    usage_events: Final = tuple(
+        usage
+        for result in results
+        if (event := as_str_mapping(result)) is not None
+        and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
+        and (usage := as_str_mapping(event.get("usage"))) is not None
+    )
+    input_seconds: Final = sum(
+        seconds
+        for usage in usage_events
+        if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
+    )
+    output_seconds: Final = sum(
+        output_duration
+        for usage in usage_events
+        if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
+    )
+    return float(input_seconds), float(output_seconds)
+
+
+def _translation_per_second_rates(
+    models: Sequence[str | None], custom_llm_provider: str
+) -> tuple[float | None, float | None]:
+    model_infos: Final = tuple(
+        _lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
+    )
+    return (
+        _first_declared_transcription_rate(("input_cost_per_second",), model_infos),
+        _first_declared_transcription_rate(("output_cost_per_second",), model_infos),
+    )
+
+
+def handle_realtime_translation_cost_calculation(
+    results: OpenAIRealtimeStreamList,
+    custom_llm_provider: str,
+    litellm_model_name: str,
+    deployment_pricing_models: Sequence[str | None] = (),
+    session_model: str | None = None,
+) -> float:
+    input_seconds, output_seconds = _translation_session_seconds(results)
+    if input_seconds <= 0 and output_seconds <= 0:
+        return 0.0
+    input_rate, output_rate = _translation_per_second_rates(
+        (*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
+    )
+    return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
+
+
 def handle_realtime_transcription_cost_calculation(
     results: OpenAIRealtimeStreamList,
     custom_llm_provider: str,
@@ -3912,17 +4048,16 @@ def _get_transcription_model_name_from_results(
     )
 
 
-_TranscriptionRateKey: TypeAlias = Literal[
-    "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
-]
-
-
-def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
+def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
     match key:
         case "input_cost_per_second":
             return info.get("input_cost_per_second")
+        case "output_cost_per_second":
+            return info.get("output_cost_per_second")
         case "input_cost_per_audio_token":
             return info.get("input_cost_per_audio_token")
+        case "output_cost_per_audio_token":
+            return info.get("output_cost_per_audio_token")
         case "input_cost_per_token":
             return info.get("input_cost_per_token")
         case "output_cost_per_token":
@@ -3931,7 +4066,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
             assert_never(key)
 
 
-def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
+def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
     """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
     because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
     if info is None:
@@ -3949,15 +4084,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
     )
 
 
-def _transcription_rate(
-    keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
-) -> float:
-    rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
-    return next((rate for rate in rates if rate is not None), 0.0)
+def _first_declared_transcription_rate(
+    keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
+) -> float | None:
+    rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
+    return next((rate for rate in rates if rate is not None), None)
+
+
+def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
+    return _first_declared_transcription_rate(keys, infos) or 0.0
 
 
 def _transcription_usage_cost(
-    usage: dict,
+    usage: Mapping[str, object],
     model_info: ModelInfo | None,
     override_info: ModelInfo | None = None,
 ) -> float:
@@ -3966,13 +4105,13 @@ def _transcription_usage_cost(
 
     usage_type: Final = usage.get("type")
     if usage_type == "duration":
-        seconds: Final = usage.get("seconds") or 0.0
+        seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
         return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
     if usage_type == "tokens":
-        input_token_details: Final = usage.get("input_token_details") or {}
-        audio_tokens: Final = input_token_details.get("audio_tokens") or 0
-        text_tokens: Final = input_token_details.get("text_tokens") or 0
-        output_tokens: Final = usage.get("output_tokens") or 0
+        input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
+        audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
+        text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
+        output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
         audio_cost: Final = float(audio_tokens) * _transcription_rate(
             ("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
         )

litellm/decisions/call.py

@@ -13,6 +13,7 @@ from typing_extensions import assert_never
 import litellm
 from litellm.cost_calculator import cost_map_key_prices_anything
 from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
+from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
 from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
@@ -555,7 +556,7 @@ def prepare_call(
         resolved_api_base, static_api_key = provider_config.resolve_connection(
             dynamic_api_base or call_api_base, dynamic_api_key or api_key
         )
-    except ValueError as error:
+    except (ValueError, httpx.InvalidURL) as error:
         raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
     if resolved_api_base is None:
         raise litellm.BadRequestError(
@@ -613,6 +614,9 @@ def prepare_call(
             litellm_params={
                 "litellm_call_id": kwargs.get("litellm_call_id"),
                 "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
+                "input_cost_per_query": (
+                    kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
+                ),
             },
             custom_llm_provider=provider,
         )
@@ -787,7 +791,9 @@ def _formatted_response(
         case DecisionsRequestBody():
             return ir_to_systemone_response(caller_response, caller_request)
         case OpenAIDecisionRequestBody():
-            return ir_to_openai_response(caller_response, caller_request, call.requested_model)
+            return ir_to_openai_response(
+                caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
+            )
         case _:
             assert_never(call.request)
 

litellm/images/main.py

@@ -28,8 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
-from litellm.llms.custom_llm import CustomLLM
-from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
+from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params
 
 #################### Initialize provider clients ####################
 llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@@ -608,10 +607,7 @@ def image_generation(
             )
         elif custom_llm_provider in litellm._custom_providers:  # Assume custom LLM provider
             # Get the Custom Handler
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -915,10 +911,7 @@ def image_edit(
 
         # Check for custom provider
         if custom_llm_provider in litellm._custom_providers:
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -995,10 +988,15 @@ def image_edit(
             or custom_llm_provider == "azure"
             or custom_llm_provider in litellm.openai_compatible_providers
         ):
+            dropped_form_params: Final = header_auth_params(custom_llm_provider)
             image_edit_request_params.update(
                 flatten_form_field_values(
-                    non_default_params,
-                    extra_body if isinstance(extra_body, dict) else None,
+                    {key: value for key, value in non_default_params.items() if key not in dropped_form_params},
+                    (
+                        {key: value for key, value in extra_body.items() if key not in dropped_form_params}
+                        if isinstance(extra_body, dict)
+                        else None
+                    ),
                 )
             )
 

litellm/integrations/anthropic_cache_control_hook.py

@@ -15,6 +15,7 @@ import itertools
 import os
 import re
 from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
+from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
@@ -25,7 +26,6 @@ from litellm.exceptions import UnsupportedParamsError
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import PromptManagementClient
-from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
     cache_control_as_prompt_cache_breakpoint,
@@ -90,8 +90,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
 _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
 _CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object])
 _ATTRIBUTE_VALUE: Final = TypeAdapter(object)
+_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 
 AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
+CostMap: TypeAlias = Mapping[str, Mapping[str, object]]
+
+
+def routed_model_info(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MODEL_INFO_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap:
+    import litellm
+
+    if model_cost is not None:
+        return model_cost
+    return cast(CostMap, litellm.model_cost)  # cast-ok: the live map holds catalog entries
+
+
+def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None:
+    declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
+    return declared if isinstance(declared, bool) else None
+
+
+def routed_cost_map() -> CostMap:
+    from litellm.utils import catalog_with_runtime_registrations
+
+    return catalog_with_runtime_registrations()
+
+
+def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None:
+    return None if model_info is None else routed_cost_map()
 
 
 def _validated_object_mapping(value: object) -> dict[object, object] | None:
@@ -325,19 +357,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non
     return provider
 
 
-def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
-    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
+def supports_openai_prompt_cache_breakpoint(
+    model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool:
+    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True
 
 
-def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
+def openai_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool | None:
+    declared: Final = declared_prompt_cache_breakpoint(model_info)
+    if declared is not None:
+        return declared
+    model_cost: Final = _catalog_unless_unrouted(model_info)
     hosted_flag: Final = (
         None
         if custom_llm_provider is None
-        else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model)
+        else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost)
     )
     if hosted_flag is not None:
         return hosted_flag
-    model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
+    model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost)
     if model_map_flag is not None:
         return model_map_flag
     fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
@@ -358,13 +398,18 @@ class _DeclaredBaseModels(BaseModel):
         return tuple(name for name in candidates if isinstance(name, str) and name)
 
 
+def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None:
+    return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info)
+
+
 def deployment_prompt_cache_breakpoint_support(
     model: str, custom_llm_provider: str, litellm_params: object
 ) -> bool | None:
     """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
     declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
+    model_info: Final = routed_model_info(declared.model_info)
     supports: Final = (
-        openai_prompt_cache_breakpoint_support(name, custom_llm_provider)
+        openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info)
         for name in dict.fromkeys((*declared.names(), model))
     )
     return next((support for support in supports if support is not None), None)
@@ -381,11 +426,10 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 
 
-def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
-    import litellm
-
-    entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
-    flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
+def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None:
+    cost_map: Final = _live_cost_map_unless(model_cost)
+    entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1]))
+    flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping))
     return next((flag is True for flag in flags if flag is not None), None)
 
 
@@ -401,7 +445,7 @@ def _hosted_openai_dialect_flag(
     model: str,
     custom_llm_provider: str | None,
     resolve_provider: Callable[[str], str | None],
-    model_cost: Mapping[str, Mapping[str, object]] | None = None,
+    model_cost: CostMap | None = None,
 ) -> bool | None:
     """
     Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@@ -418,13 +462,7 @@ def _hosted_openai_dialect_flag(
     row of the same name, so an exact entry only speaks for a deployment when it is
     keyed for that deployment's provider.
     """
-    import litellm
-
-    cost_map: Final = (
-        cast(Mapping[str, Mapping[str, object]], litellm.model_cost)  # cast-ok: the live map holds catalog entries
-        if model_cost is None
-        else model_cost
-    )
+    cost_map: Final = _live_cost_map_unless(model_cost)
     exact_entry: Final = cost_map.get(model)
     exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
     if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
@@ -434,10 +472,16 @@ def _hosted_openai_dialect_flag(
         return None
     if isinstance(exact_entry, Mapping) and exact_provider == provider:
         return _hosted_entry_flag(exact_entry, lambda: provider)
+    entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider))
+    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
+    return next((flag for flag in flags if flag is not None), None)
+
+
+def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]:
     from litellm.utils import get_potential_model_names
 
     names: Final = get_potential_model_names(model, provider)
-    candidates: Final = (
+    return (
         names["combined_model_name"],
         names["region_free_combined_model_name"],
         names["split_model"],
@@ -445,9 +489,60 @@ def _hosted_openai_dialect_flag(
         names["stripped_model_name"],
         names["provider_prefixed_model_name"],
     )
-    entries: Final = (cost_map.get(candidate) for candidate in candidates)
-    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
-    return next((flag for flag in flags if flag is not None), None)
+
+
+def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool:
+    entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider)))
+    return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries)
+
+
+_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse"))
+_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle"))
+
+
+def _declared_hosted_flag(
+    declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object
+) -> bool | None:
+    """
+    A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the
+    request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes
+    a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep
+    the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass
+    the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost
+    map, or the deployment's own ``litellm_provider``, lists under that provider.
+    """
+    if provider is None or provider == "openai":
+        return None
+    return (
+        declared
+        and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT
+        and (
+            provider in _PROVIDERS_HOSTING_OPENAIS_API
+            or deployment_provider == provider
+            or _cost_map_lists_model_for_provider(model, provider, model_cost)
+        )
+        and _sends_openai_shaped_chat(model, provider)
+    )
+
+
+@lru_cache(maxsize=512)
+def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool:
+    from litellm.exceptions import BadRequestError
+    from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
+    from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig
+    from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+    from litellm.llms.openai.openai import OpenAIConfig
+    from litellm.types.utils import LlmProviders
+    from litellm.utils import ProviderConfigManager
+
+    try:
+        served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider)
+        chat_config: Final = ProviderConfigManager.get_provider_chat_config(
+            model=served_model, provider=LlmProviders(provider)
+        )
+    except (BadRequestError, ValueError):
+        return False
+    return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig))
 
 
 def targets_openai_api(api_base: object) -> bool:
@@ -556,12 +651,6 @@ def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | N
     )
 
 
-def _reads_prompt_cache_breakpoint_natively(model: str, custom_llm_provider: str | None) -> bool:
-    return supports_openai_prompt_cache_breakpoint(
-        model, custom_llm_provider
-    ) and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
-
-
 def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
     provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
     return provider if isinstance(provider, str) else None
@@ -770,24 +859,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message")
 
-        custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0])
+        request_params: Final = cast(Mapping[str, object], non_default_params)  # cast-ok: base hook's bare dict
+        custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
         stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
+        model_info: Final = routed_model_info(request_params.get("model_info"))
         openai_dialect: Final = (
             stamped_dialect
             if isinstance(stamped_dialect, bool)
             else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
                 model,
                 custom_llm_provider,
-                non_default_params.get("api_base") or non_default_params.get("base_url"),
-                non_default_params.get("prompt_cache_options"),
+                request_params.get("api_base") or request_params.get("base_url"),
+                request_params.get("prompt_cache_options"),
+                model_info,
             )
         )
         processed_messages = copy.deepcopy(
-            messages
-            if openai_dialect
-            or not message_points
-            or _reads_prompt_cache_breakpoint_natively(model, custom_llm_provider)
-            else [without_prompt_cache_breakpoints(message) for message in messages]
+            [without_prompt_cache_breakpoints(message) for message in messages]
+            if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
+            else messages
         )
         # A provisional message list defers every role-targeted point to the pass holding
         # the final one: a role with no message here may have one there, and settling all
@@ -857,32 +947,35 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object = None,
         prompt_cache_options: object = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> bool:
         if model is None:
             return False
-        hosted_flag: Final = _hosted_openai_dialect_flag(
-            model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
+        declared: Final = declared_prompt_cache_breakpoint(model_info)
+        hosted_flag: Final = (
+            _hosted_openai_dialect_flag(
+                model,
+                custom_llm_provider,
+                lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base),
+                _catalog_unless_unrouted(model_info),
+            )
+            if declared is None
+            else _declared_hosted_flag(
+                declared,
+                model,
+                custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base),
+                routed_cost_map(),
+                None if model_info is None else model_info.get("litellm_provider"),
+            )
         )
         if hosted_flag is not None:
             return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
-        if not supports_openai_prompt_cache_breakpoint(model):
+        if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
             return False
         if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
             return False
         return prompt_cache_options is not None or targets_openai_api(api_base)
 
-    @staticmethod
-    def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
-        return (
-            _hosted_openai_dialect_flag(
-                model,
-                custom_llm_provider,
-                AnthropicCacheControlHook._resolve_provider,
-                GetModelCostMap.loaded_model_cost_map(),
-            )
-            is True
-        )
-
     @staticmethod
     def _resolve_provider(model: str, api_base: object = None) -> str | None:
         return _served_provider_for_model(model, api_base)
@@ -1390,6 +1483,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object,
         prompt_cache_options: object,
+        model_info: Mapping[str, object] | None = None,
     ) -> Sequence[Mapping[str, object]]:
         """Carry onto the points what the prompt-management hook never receives.
 
@@ -1401,7 +1495,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         across requests.
         """
         with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect(
-            points, model, custom_llm_provider, api_base, prompt_cache_options
+            points, model, custom_llm_provider, api_base, prompt_cache_options, model_info
         )
         stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect)
         if external_breakpoints == 0 and not stamped_earlier:
@@ -1415,14 +1509,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object,
         prompt_cache_options: object,
+        model_info: Mapping[str, object] | None = None,
     ) -> Sequence[Mapping[str, object]]:
-        if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider):
+        if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(
+            model, custom_llm_provider
+        ):
             return points
         return AnthropicCacheControlHook._stamped(
             points,
             "_litellm_openai_dialect",
             AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-                model, custom_llm_provider, api_base, prompt_cache_options
+                model, custom_llm_provider, api_base, prompt_cache_options, model_info
             ),
         )
 
@@ -1481,6 +1578,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         cache_control: object = None,
         request_kwargs: object = None,
         on_messages_route: bool = False,
+        model_info: Mapping[str, object] | None = None,
     ) -> list[CacheControlInjectionPoint]:
         """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
 
@@ -1498,7 +1596,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True:
             return []
 
-        if not supports_anthropic_cache_control(model, custom_llm_provider):
+        if not supports_anthropic_cache_control(model, custom_llm_provider, model_info):
             return []
 
         if AnthropicCacheControlHook._request_has_cache_control(
@@ -1576,6 +1674,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         tools: list | None = None,
         enable_prompt_caching: bool | None = None,
         api_base: object = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> None:
         """For /chat/completions: resolve the injection points the request should carry.
 
@@ -1600,6 +1699,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 custom_llm_provider,
                 api_base,
                 non_default_params.get("prompt_cache_options"),
+                model_info,
             )
             non_default_params["cache_control_injection_points"] = (
                 AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider)
@@ -1616,6 +1716,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             enable_prompt_caching=enable_prompt_caching,
             cache_control=non_default_params.get("cache_control"),
             request_kwargs=non_default_params,
+            model_info=model_info,
         )
         if points:
             non_default_params["cache_control_injection_points"] = points
@@ -1734,6 +1835,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 cache_control=cache_control,
                 request_kwargs=kwargs,
                 on_messages_route=True,
+                model_info=routed_model_info(kwargs.get("model_info")),
             )
             if model is not None
             else ()
@@ -1742,7 +1844,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             return messages, system
 
         openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-            model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
+            model,
+            custom_llm_provider,
+            api_base,
+            kwargs.get("prompt_cache_options"),
+            routed_model_info(kwargs.get("model_info")),
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
         messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(

litellm/integrations/custom_guardrail.py

@@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
         enrich_http_exception_with_guardrail_context(result, self)
         raise result
 
+    def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
+        return (
+            isinstance(request_data.get("guardrails"), list)
+            and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
+        )
+
     async def async_post_call_success_deployment_hook(
         self,
         request_data: dict,
@@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
         """
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
-        # should run guardrail
-        litellm_guardrails: Final = request_data.get("guardrails")
-        if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
-            return None
-
-        if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
+        if not self.runs_post_call_deployment_hook(request_data):
             return None
 
         target: Final = self._deployment_hook_target()

litellm/integrations/websearch_interception/handler.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
 from typing_extensions import Never, ReadOnly
 
 import litellm
+from litellm._internal_context import billed_separately
 from litellm._logging import verbose_logger
 from litellm.anthropic_interface import messages as anthropic_messages
 from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
             search_kwargs: Final = MappingProxyType(
                 {**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
             )
-            result: Final = (
-                await litellm.asearch(
-                    query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+            with billed_separately():
+                result: Final = (
+                    await litellm.asearch(
+                        query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+                    )
+                    if search_metadata is None
+                    else await litellm.asearch(
+                        query=query_arg,
+                        search_provider=search_provider,
+                        litellm_metadata=search_metadata,
+                        **_NO_ASEARCH_NAMED,
+                        **search_kwargs,
+                    )
                 )
-                if search_metadata is None
-                else await litellm.asearch(
-                    query=query_arg,
-                    search_provider=search_provider,
-                    litellm_metadata=search_metadata,
-                    **_NO_ASEARCH_NAMED,
-                    **search_kwargs,
-                )
-            )
 
             # Format using transformation function
             search_result_text: Final = WebSearchTransformation.format_search_response(result)

litellm/litellm_core_utils/allowed_tools_tool_choice.py

@@ -0,0 +1,111 @@
+from collections.abc import Mapping
+from typing import Final, Literal
+
+from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+from litellm.exceptions import BadRequestError
+
+ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
+
+CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
+    '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
+)
+
+
+class ChatAllowedTools(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    mode: Literal["auto", "required"]
+    tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
+
+
+class ChatAllowedToolsToolChoice(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    type: Literal["allowed_tools"]
+    allowed_tools: ChatAllowedTools
+
+
+_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
+
+
+_SELECTOR: Final = TypeAdapter(dict[str, object])
+
+
+class _ToolChoiceType(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str
+
+
+_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType)
+
+
+def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
+    try:
+        return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools"
+    except ValidationError:
+        return False
+
+
+def raise_if_allowed_tools_tool_choice_unsupported(
+    tool_choice: object,
+    custom_llm_provider: str,
+    model: str,
+) -> None:
+    """Providers outside the set would silently send the request with no tool restriction at all, so the call stops
+    here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
+    if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
+        return
+    raise BadRequestError(
+        message=(
+            "tool_choice type 'allowed_tools' is only supported for "
+            f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
+            "drop_params does not drop it, since that would let the model call tools the request excluded. "
+            "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
+        ),
+        model=model,
+        llm_provider=custom_llm_provider,
+    )
+
+
+def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
+    try:
+        return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
+    except ValidationError:
+        return None
+
+
+def _has_name(selector: Mapping[str, object]) -> bool:
+    name: Final = selector.get("name")
+    return isinstance(name, str) and name != ""
+
+
+def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
+    tool_type: Final = tool.get("type")
+    if tool_type not in ("function", "custom"):
+        return tool
+    if tool_type not in tool:
+        return tool if _has_name(tool) else None
+    try:
+        nested: Final = _SELECTOR.validate_python(tool[tool_type])
+    except ValidationError:
+        return None
+    outer: Final = {key: value for key, value in tool.items() if key != tool_type}
+    if not _has_name(nested) or outer.keys() & nested.keys():
+        return None
+    return {**outer, **nested}
+
+
+def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None:
+    """The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is malformed or
+    carries conflicting keys, so the caller sends the original rather than a rewrite that could mean something looser."""
+    flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools)
+    if any(tool is None for tool in flattened):
+        return None
+    return ToolChoiceAllowedParam(
+        type="allowed_tools",
+        mode=choice.allowed_tools.mode,
+        tools=[dict(tool) for tool in flattened if tool is not None],
+    )

litellm/litellm_core_utils/audio_utils/transcription_streaming.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import datetime
+import json
+import traceback
+from collections.abc import AsyncIterator, Callable, Iterator, Mapping
+from typing import Final, Protocol
+
+import anyio
+from openai import AsyncStream, Stream
+from openai.types.audio import (
+    TranscriptionStreamEvent,
+    TranscriptionTextDeltaEvent,
+    TranscriptionTextDoneEvent,
+)
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    TranscriptionUsageObjectTransformation,
+)
+from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
+
+_STREAM_FLAG: Final = TypeAdapter(bool)
+
+
+def transcription_stream_flag(value: object) -> bool | None:
+    try:
+        return _STREAM_FLAG.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def transcription_stream_unset(value: object) -> bool:
+    return value is None or value == ""
+
+
+def transcription_stream_requested(value: object) -> bool:
+    return transcription_stream_flag(value) is True
+
+
+TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
+
+TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
+
+
+def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
+    from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
+
+    model: Final = call_details.get("model")
+    custom_llm_provider: Final = call_details.get("custom_llm_provider")
+    extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
+
+    def map_exception(exception: Exception) -> Exception:
+        try:
+            mapped: Final[object] = exception_type(
+                model=model if isinstance(model, str) else None,
+                original_exception=exception,
+                custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+                extra_kwargs=extra_kwargs,
+            )
+        except Exception as mapped_exception:  # noqa: BLE001  # exception_type raises the mapped exception
+            return mapped_exception
+        return mapped if isinstance(mapped, Exception) else exception
+
+    return map_exception
+
+
+def _guarded_transcription_event_payloads(
+    response: TranscriptionResponse,
+) -> tuple[dict[str, object], dict[str, object]]:
+    text: Final = response.text or ""
+    usage: Final = response.usage
+    languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
+    return (
+        {"type": "transcript.text.delta", "delta": text},
+        {
+            "type": "transcript.text.done",
+            "text": text,
+            **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
+            **({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
+        },
+    )
+
+
+def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
+    return (
+        *(
+            f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
+            for event in _guarded_transcription_event_payloads(response)
+        ),
+        TRANSCRIPTION_STREAM_DONE_FRAME,
+    )
+
+
+class GuardedTranscriptionReplay:
+    def __init__(self, response: TranscriptionResponse) -> None:
+        self.response: Final = response
+        self._hidden_params: dict[str, object] = response.hidden_params
+
+    def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
+        return self._events()
+
+    async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
+        delta, done = _guarded_transcription_event_payloads(self.response)
+        yield TranscriptionTextDeltaEvent.model_construct(**delta)
+        yield TranscriptionTextDoneEvent.model_construct(**done)
+
+
+class TranscriptionStreamLogging(Protocol):
+    def success_handler(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    async def async_success_handler(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def handle_sync_success_callbacks_for_async_calls(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def failure_handler(
+        self,
+        exception: Exception,
+        traceback_exception: str,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+        *,
+        for_async_request: bool = False,
+    ) -> None: ...
+
+    async def async_failure_handler(
+        self,
+        exception: Exception,
+        traceback_exception: str,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
+
+    def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
+
+
+class _TranscriptionEventCollector:
+    def __init__(self, duration: float | None) -> None:
+        self.duration = duration
+        self.text_deltas: list[str] = []  # mutable-ok: streaming deltas accumulate until the terminal event
+        self.done_event: TranscriptionTextDoneEvent | None = None
+
+    def add(self, event: TranscriptionStreamEvent) -> None:
+        if isinstance(event, TranscriptionTextDeltaEvent):
+            self.text_deltas.append(event.delta)
+        elif isinstance(event, TranscriptionTextDoneEvent):
+            self.done_event = event
+
+    def response(self) -> TranscriptionResponse:
+        done_event: Final = self.done_event
+        response: Final = TranscriptionResponse(
+            text=done_event.text if done_event is not None else "".join(self.text_deltas),
+            usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
+            languages=getattr(done_event, "languages", None) if done_event is not None else None,
+        )
+        if self.duration is not None:
+            response.set_audio_transcription_duration(self.duration)
+        return response
+
+
+def _billed_usage(response: TranscriptionResponse) -> Usage:
+    usage: Final[object] = getattr(response, "usage", None)
+    token_usage: Final = (
+        TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
+        if isinstance(usage, TranscriptionUsageTokensObject)
+        else None
+    )
+    return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+
+
+def _record_billable_partial_spend(
+    logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
+) -> None:
+    partial_response: Final = collector.response()
+    partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
+    if partial_cost <= 0:
+        return
+    logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
+
+
+class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
+    def __init__(
+        self,
+        stream: Stream[TranscriptionStreamEvent],
+        logging_obj: TranscriptionStreamLogging,
+        start_time: datetime.datetime,
+        map_exception: TranscriptionStreamExceptionMapper,
+    ) -> None:
+        self.__dict__.update(stream.__dict__)
+        self._hidden_params: dict[str, object] = {  # mutable-ok: ResponseMetadata fills in the call's metadata
+            "additional_headers": process_response_headers(stream.response.headers)
+        }
+        self._logging_obj = logging_obj
+        self._start_time = start_time
+        self._map_exception = map_exception
+        self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
+        self._finalized = False
+        self._failed = False
+        source_iterator: Final = self._iterator
+        self._iterator = self._logging_iterator(source_iterator)
+
+    def _logging_iterator(
+        self, source_iterator: Iterator[TranscriptionStreamEvent]
+    ) -> Iterator[TranscriptionStreamEvent]:
+        try:
+            for event in source_iterator:
+                self._collector.add(event)
+                yield event
+        except Exception as exception:
+            self._failed = True
+            _record_billable_partial_spend(self._logging_obj, self._collector)
+            mapped_exception: Final = self._map_exception(exception)
+            end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+            self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
+            raise mapped_exception from exception
+        finally:
+            self._finalize()
+
+    def _finalize(self) -> None:
+        if self._finalized or self._failed:
+            return
+        self._finalized = True
+        self._logging_obj.success_handler(
+            self._collector.response(),
+            self._start_time,
+            datetime.datetime.now(),  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+        )
+
+    def close(self) -> None:
+        try:
+            super().close()
+        finally:
+            self._finalize()
+
+    @property
+    def logging_obj(self) -> TranscriptionStreamLogging:
+        return self._logging_obj
+
+
+class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
+    def __init__(
+        self,
+        stream: AsyncStream[TranscriptionStreamEvent],
+        logging_obj: TranscriptionStreamLogging,
+        start_time: datetime.datetime,
+        map_exception: TranscriptionStreamExceptionMapper,
+    ) -> None:
+        self.__dict__.update(stream.__dict__)
+        self._hidden_params: dict[str, object] = {  # mutable-ok: ResponseMetadata fills in the call's metadata
+            "additional_headers": process_response_headers(stream.response.headers)
+        }
+        self._logging_obj = logging_obj
+        self._start_time = start_time
+        self._map_exception = map_exception
+        self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
+        self._finalized = False
+        self._failed = False
+        source_iterator: Final = self._iterator
+        self._iterator = self._logging_iterator(source_iterator)
+
+    async def _logging_iterator(
+        self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
+    ) -> AsyncIterator[TranscriptionStreamEvent]:
+        try:
+            async for event in source_iterator:
+                self._collector.add(event)
+                yield event
+        except Exception as exception:
+            self._failed = True
+            _record_billable_partial_spend(self._logging_obj, self._collector)
+            mapped_exception: Final = self._map_exception(exception)
+            end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+            self._logging_obj.failure_handler(
+                mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
+            )
+            await self._logging_obj.async_failure_handler(
+                mapped_exception,
+                traceback.format_exc(),
+                self._start_time,
+                end_time,
+            )
+            raise mapped_exception from exception
+        finally:
+            await self._finalize()
+
+    async def _finalize(self) -> None:
+        if self._finalized or self._failed:
+            return
+        self._finalized = True
+        response: Final = self._collector.response()
+        end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+        self._logging_obj.handle_sync_success_callbacks_for_async_calls(
+            result=response,
+            start_time=self._start_time,
+            end_time=end_time,
+        )
+        with anyio.CancelScope(shield=True):
+            await self._logging_obj.async_success_handler(
+                result=response,
+                start_time=self._start_time,
+                end_time=end_time,
+            )
+
+    async def close(self) -> None:
+        try:
+            await super().close()
+        finally:
+            await self._finalize()
+
+    @property
+    def logging_obj(self) -> TranscriptionStreamLogging:
+        return self._logging_obj
+
+
+def wrap_transcription_stream(
+    stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
+    logging_obj: TranscriptionStreamLogging,
+    start_time: datetime.datetime,
+    map_exception: TranscriptionStreamExceptionMapper,
+) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
+    if isinstance(stream, AsyncStream):
+        return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
+    return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)

litellm/litellm_core_utils/audio_utils/utils.py

@@ -21,6 +21,16 @@ from litellm.types.files import (
 from litellm.types.utils import FileTypes
 
 
+def normalized_audio_duration_seconds(value: object) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    try:
+        seconds: Final = float(value)
+    except OverflowError:
+        return None
+    return seconds if math.isfinite(seconds) and seconds >= 0 else None
+
+
 @dataclass
 class ProcessedAudioFile:
     """

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -8,6 +8,7 @@ from typing import Final
 
 from pydantic import TypeAdapter, ValidationError
 
+from litellm._internal_context import internal_sub_call
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@@ -204,12 +205,13 @@ async def _execute_chat_completion_agentic_plan(
     )
 
     try:
-        response_followup = await litellm.acompletion(
-            model=full_model_name,
-            messages=patch.messages,
-            **optional_params_for_followup,
-            **kwargs_for_followup,
-        )
+        with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None):
+            response_followup = await litellm.acompletion(
+                model=full_model_name,
+                messages=patch.messages,
+                **optional_params_for_followup,
+                **kwargs_for_followup,
+            )
         if _post_hook_overridden(callback):
             try:
                 response_followup = await callback.async_post_agentic_loop_response_hook(

litellm/litellm_core_utils/core_helpers.py

@@ -97,7 +97,7 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
     return float(seconds / denominator)
 
 
-_DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
+_LAX_BOOL: Final = TypeAdapter(bool)
 
 
 def as_str_mapping(value: object) -> Mapping[str, object] | None:
@@ -160,15 +160,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]:
     return iter_items(value) if value else ()
 
 
-def normalize_drop_params(value: object) -> bool | None:
+def lax_bool(value: object) -> bool | None:
     if value is None or isinstance(value, bool):
         return value
     try:
-        return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
+        return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
     except ValidationError:
         return None
 
 
+def normalize_drop_params(value: object) -> bool | None:
+    return lax_bool(value)
+
+
 def drops_unsupported_params(drop_params: object) -> bool:
     return litellm.drop_params is True or normalize_drop_params(drop_params) is True
 

litellm/litellm_core_utils/decisions_pricing.py

@@ -12,6 +12,7 @@ from typing import Final
 
 DECISIONS_PRICING_SUFFIX: Final = "_decisions"
 DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
+QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
 
 
 def is_decisions_call_type(call_type: str) -> bool:
@@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
     }
 
 
+def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
+    if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
+        return None
+    rates: Final = {**model_info, **overrides}
+    request_price: Final = rates.get("input_cost_per_query")
+    if not isinstance(request_price, (int, float)):
+        return None
+    bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token"))
+    return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price)
+
+
 def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
     """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
     plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -107,6 +107,7 @@ class ExceptionCheckers:
             "exceeds the available context size",  # llama.cpp/Lemonade
             "exceeds the maximum number of tokens allowed",  # Gemini
             '"code": "context_length_exceeded"',
+            "prompt is too long",
         ]
         for substring in known_exception_substrings:
             if substring in _error_str_lowercase:

litellm/litellm_core_utils/litellm_logging.py

@@ -22,6 +22,8 @@ from types import MappingProxyType, TracebackType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
 
 from httpx import Response, Timeout
+from openai import AsyncStream as OpenAIAsyncStream
+from openai import Stream as OpenAIStream
 from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
@@ -2130,6 +2132,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if resolved_cache_hit is True:
             return 0.0
 
+        if isinstance(result, (OpenAIStream, OpenAIAsyncStream)):
+            return None
+
         if is_unbilled_non_inference_call(
             self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
         ):
@@ -3625,6 +3630,7 @@ class Logging(LiteLLMLoggingBaseClass):
         verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
         if not self.should_run_logging(event_type="sync_success"):  # prevent double logging
             return
+        provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
         start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
             start_time=start_time,
             end_time=end_time,
@@ -3639,7 +3645,12 @@ class Logging(LiteLLMLoggingBaseClass):
         try:
             ## BUILD COMPLETE STREAMED RESPONSE
             complete_streaming_response: (
-                ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
+                ModelResponse
+                | TextCompletionResponse
+                | ResponsesAPIResponse
+                | InteractionsAPIResponse
+                | TranscriptionResponse
+                | None
             ) = None
             if "complete_streaming_response" in self.model_call_details and not for_async_request:
                 return  # break out of this.
@@ -3655,7 +3666,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["complete_streaming_response"] = complete_streaming_response
                 self._surface_response_headers_from_result(complete_streaming_response)
                 self.model_call_details["response_cost"] = self.response_cost_calculator(
-                    result=complete_streaming_response
+                    result=complete_streaming_response if provider_transcription is None else provider_transcription
                 )
                 self._add_continued_partial_spend(complete_streaming_response)
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
@@ -3665,7 +3676,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 ## STANDARDIZED LOGGING PAYLOAD
                 if not for_async_request:
                     self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
-                        logged_streaming_response, start_time, end_time
+                        logged_streaming_response if provider_transcription is None else provider_transcription,
+                        start_time,
+                        end_time,
                     )
                     standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
                         "standard_logging_object"
@@ -4192,6 +4205,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
             )
         )
+        provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
         start_time, end_time, result = self._success_handler_helper_fn(
             start_time=start_time,
             end_time=end_time,
@@ -4211,9 +4225,7 @@ class Logging(LiteLLMLoggingBaseClass):
         ## BUILD COMPLETE STREAMED RESPONSE
         if "async_complete_streaming_response" in self.model_call_details:
             return  # break out of this.
-        complete_streaming_response: Final[
-            ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
-        ] = self._get_assembled_streaming_response(
+        complete_streaming_response: Final = self._get_assembled_streaming_response(
             result=result,
             start_time=start_time,
             end_time=end_time,
@@ -4235,7 +4247,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     get_base_model_from_metadata(model_call_details=self.model_call_details)
                     # base_model defaults to None if not set on model_info
                     self.model_call_details["response_cost"] = self.response_cost_calculator(
-                        result=complete_streaming_response
+                        result=complete_streaming_response if provider_transcription is None else provider_transcription
                     )
 
                 verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"])
@@ -4262,7 +4274,9 @@ class Logging(LiteLLMLoggingBaseClass):
             ## STANDARDIZED LOGGING PAYLOAD
             try:
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
-                    logged_streaming_response, start_time, end_time
+                    logged_streaming_response if provider_transcription is None else provider_transcription,
+                    start_time,
+                    end_time,
                 )
             except Exception:  # noqa: BLE001  # payload build must never block later callbacks (slot release)
                 verbose_logger.exception(
@@ -5132,10 +5146,17 @@ class Logging(LiteLLMLoggingBaseClass):
         end_time: datetime.datetime,
         is_async: bool,
         streaming_chunks: list[object],
-    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
+    ) -> (
+        ModelResponse
+        | TextCompletionResponse
+        | ResponsesAPIResponse
+        | InteractionsAPIResponse
+        | TranscriptionResponse
+        | None
+    ):
         if self.stream is not True:
             return None
-        if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
+        if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)):
             return result
         elif isinstance(
             result,

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost(
 class PromptTokensDetailsResult(TypedDict):
     cache_hit_tokens: int
     cache_hit_audio_tokens: ReadOnly[int]
+
+    cached_text_tokens: ReadOnly[int]
+    cached_audio_tokens: ReadOnly[int]
+    cached_image_tokens: ReadOnly[int]
+    has_cached_tokens_details: ReadOnly[bool]
     cache_creation_tokens: int
     cache_creation_token_details: CacheCreationTokenDetails | None
     text_tokens: int
@@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
     return PromptTokensDetailsResult(
         cache_hit_tokens=cache_hit_tokens,
         cache_hit_audio_tokens=cached_audio_tokens,
+        cached_text_tokens=cached_text_tokens,
+        cached_audio_tokens=cached_audio_tokens,
+        cached_image_tokens=cached_image_tokens,
+        has_cached_tokens_details=cached_tokens_details is not None,
         cache_creation_tokens=cache_creation_tokens,
         cache_creation_token_details=cache_creation_token_details,
         text_tokens=text_tokens,
@@ -1234,15 +1243,11 @@ def _calculate_input_cost(
     prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
 
     ### CACHE READ COST - Now uses tiered pricing
-    cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
-    audio_cache_read_rate: Final = get_cost_per_unit(
-        model_info,
-        get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
-        None,
-    )
-    prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
-    prompt_cost += float(cache_hit_audio_tokens) * (
-        audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
+    prompt_cost += _calculate_cache_read_cost(
+        prompt_tokens_details=prompt_tokens_details,
+        model_info=model_info,
+        cache_read_cost=cache_read_cost,
+        service_tier=service_tier,
     )
 
     ### AUDIO COST
@@ -1331,6 +1336,38 @@ def _calculate_input_cost(
     return prompt_cost
 
 
+def _calculate_cache_read_cost(
+    prompt_tokens_details: PromptTokensDetailsResult,
+    model_info: ModelInfo,
+    cache_read_cost: float,
+    service_tier: str | None,
+) -> float:
+    cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"]
+    cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"]
+    cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"]
+    classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
+    unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0)
+    total_cost = (  # rebind-ok: cached modality components accumulate into one cache-read cost
+        float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost
+    )
+
+    if cached_audio_tokens:
+        cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier)
+        cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost)
+        total_cost += (  # rebind-ok: cached audio contributes to cache-read cost
+            float(cached_audio_tokens) * float(cached_audio_cost or 0.0)
+        )
+
+    if cached_image_tokens:
+        cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier)
+        cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost)
+        total_cost += (  # rebind-ok: cached images contribute to cache-read cost
+            float(cached_image_tokens) * float(cached_image_cost or 0.0)
+        )
+
+    return total_cost
+
+
 def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
     """
     Resolve the per-model regional-processing uplift multiplier for a given
@@ -1502,6 +1539,10 @@ def generic_cost_per_token(
     prompt_tokens_details = PromptTokensDetailsResult(
         cache_hit_tokens=0,
         cache_hit_audio_tokens=0,
+        cached_text_tokens=0,
+        cached_audio_tokens=0,
+        cached_image_tokens=0,
+        has_cached_tokens_details=False,
         cache_creation_tokens=0,
         cache_creation_token_details=None,
         text_tokens=usage.prompt_tokens,
@@ -1680,6 +1721,7 @@ class BilledTokenRates:
     cache_creation_input_token_cost: float
     cache_creation_input_token_cost_by_ttl: Mapping[str, float]
     output_cost_per_reasoning_token: float
+    cache_read_input_image_token_cost: float | None = None
 
     def scaled(self, multiplier: float) -> "BilledTokenRates":
         if multiplier == 1.0:
@@ -1694,6 +1736,11 @@ class BilledTokenRates:
                 {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
             ),
             output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
+            cache_read_input_image_token_cost=(
+                self.cache_read_input_image_token_cost * multiplier
+                if self.cache_read_input_image_token_cost is not None
+                else None
+            ),
         )
 
 
@@ -1802,6 +1849,11 @@ def _cost_map_billed_rates(
         cache_creation_input_token_cost=base_costs.cache_creation_rate,
         cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
         output_cost_per_reasoning_token=reasoning_rate,
+        cache_read_input_image_token_cost=_get_cost_per_unit(
+            model_info,
+            _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier),
+            None,
+        ),
     ).scaled(multiplier)
 
 
@@ -1882,6 +1934,12 @@ def get_token_type_cost_breakdown(
     cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
         usage
     )
+    cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"]
+    image_cache_read_rate: Final = (
+        rates.cache_read_input_image_token_cost
+        if rates.cache_read_input_image_token_cost is not None
+        else rates.cache_read_input_token_cost
+    )
     cache_creation_cost: Final = (
         float(cache_creation_tokens) * rates.cache_creation_input_token_cost
         if custom_cost_per_token is not None
@@ -1895,8 +1953,9 @@ def get_token_type_cost_breakdown(
     return TokenTypeCostBreakdown(
         reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
         cache_read_cost=(
-            float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
+            float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost
             + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
+            + float(cached_image_tokens) * image_cache_read_rate
         ),
         cache_creation_cost=cache_creation_cost,
         rates=rates,

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -970,9 +970,11 @@ def convert_to_model_response_object(
             model_response_object.data = response_object["data"]
 
             if "usage" in response_object and response_object["usage"] is not None:
-                model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
-                model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
-                model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
+                embedding_usage: Final = model_response_object.usage or Usage()
+                embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
+                embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
+                embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0)
+                model_response_object.usage = embedding_usage
 
             if start_time is not None and end_time is not None:
                 model_response_object._response_ms = (
@@ -1015,6 +1017,10 @@ def convert_to_model_response_object(
                 if key in response_object:
                     setattr(model_response_object, key, response_object[key])
 
+            detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages"))
+            if detected_languages is not None:
+                model_response_object.languages = detected_languages
+
             if "usage" in response_object and response_object["usage"] is not None:
                 tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None
 

litellm/litellm_core_utils/model_param_helper.py

@@ -1,6 +1,6 @@
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import ClassVar, Final
+from typing import ClassVar, Final, get_args
 
 from openai.types.chat.completion_create_params import (
     CompletionCreateParamsNonStreaming,
@@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.anthropic import AnthropicMessagesRequest
+from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
 from litellm.types.rerank import RerankRequest
 
 
@@ -178,6 +179,7 @@ class ModelParamHelper:
 
         This follows the OpenAI API Spec
         """
+        litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams))
         try:
             from openai.types.audio.transcription_create_params import (
                 TranscriptionCreateParamsNonStreaming,
@@ -187,11 +189,11 @@ class ModelParamHelper:
             non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys())
             streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys())
 
-            all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs)
+            all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs)
             return all_transcription_kwargs
         except Exception as e:
             verbose_logger.debug("Error getting transcription kwargs %s", str(e))
-            return set()
+            return litellm_transcription_kwargs
 
     @staticmethod
     def _get_litellm_supported_responses_api_kwargs() -> set[str]:

litellm/litellm_core_utils/pattern_match_deployments.py

@@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching
 import copy
 import itertools
 import re
+from collections.abc import Mapping
 from re import Match
 from typing import Final
 
@@ -22,6 +23,16 @@ class _RoutedDeployment(BaseModel):
 
 
 _ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
+_PATTERN_WIDE_CACHE_FLAGS: Final = frozenset(("supports_prompt_cache_breakpoint", "supports_prompt_caching"))
+_AZURE_AI_PATTERN_CACHE_FLAGS: Final = frozenset(("supports_prompt_caching",))
+
+
+def _cache_flags_left_to_the_catalog(litellm_params: Mapping[str, object]) -> frozenset[str]:
+    model: Final = litellm_params.get("model")
+    provider: Final = litellm_params.get("custom_llm_provider") or inferred_provider(
+        model if isinstance(model, str) else None
+    )
+    return _AZURE_AI_PATTERN_CACHE_FLAGS if provider == "azure_ai" else _PATTERN_WIDE_CACHE_FLAGS
 
 
 class PatternUtils:
@@ -133,6 +144,11 @@ class PatternMatchRouter:
                 matched_pattern=matched_pattern,
                 litellm_deployment_litellm_model=deployment["litellm_params"]["model"],
             )
+            if "*" in deployment["litellm_params"]["model"] and isinstance(new_deployment.get("model_info"), dict):
+                left_to_the_catalog = _cache_flags_left_to_the_catalog(deployment["litellm_params"])
+                new_deployment["model_info"] = {
+                    key: value for key, value in new_deployment["model_info"].items() if key not in left_to_the_catalog
+                }
             new_deployments.append(new_deployment)
 
         return new_deployments

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -77,7 +77,7 @@ from .common_utils import (
     parse_tool_call_arguments,
     salvage_concatenated_tool_arguments,
 )
-from .image_handling import convert_url_to_base64
+from .image_handling import convert_url_to_base64, is_pdf_media_type
 
 
 def default_pt(messages):
@@ -911,8 +911,17 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) ->
         )
 
 
+def _anthropic_https_url_block(
+    image_url: str, image_format: str | None
+) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
+    source: Final = AnthropicContentParamSourceUrl(type="url", url=image_url)
+    if image_format and is_pdf_media_type(image_format):
+        return AnthropicMessagesDocumentParam(type="document", source=source)
+    return AnthropicMessagesImageParam(type="image", source=source)
+
+
 def create_anthropic_image_param(
-    image_url_input: str | dict,
+    image_url_input: str | Mapping[str, object],
     format: str | None = None,
     is_bedrock_invoke: bool = False,
 ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
@@ -921,13 +930,11 @@ def create_anthropic_image_param(
 
     Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
     """
-    # Extract URL and format from input
-    if isinstance(image_url_input, str):
-        image_url = image_url_input
-    else:
-        image_url = image_url_input.get("url", "")
-        if format is None:
-            format = image_url_input.get("format")
+    url_value: Final = image_url_input if isinstance(image_url_input, str) else image_url_input.get("url")
+    format_value: Final = None if isinstance(image_url_input, str) else image_url_input.get("format")
+    image_url: Final = url_value if isinstance(url_value, str) else ""
+    explicit_format: Final = format if format is not None else format_value
+    image_format: Final = explicit_format if isinstance(explicit_format, str) else None
 
     # Check if the image URL is an HTTP/HTTPS URL
     if image_url.startswith("http://") or image_url.startswith("https://"):
@@ -935,21 +942,14 @@ def create_anthropic_image_param(
         # as these providers don't support URL sources for images
         if is_bedrock_invoke or image_url.startswith("http://"):
             base64_url: Final = convert_url_to_base64(url=image_url)
-            image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
-            return _anthropic_content_element_factory(image_chunk)
+            image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=image_format)
+            return anthropic_content_element_factory(image_chunk)
         else:
-            # HTTPS URL - pass directly for regular Anthropic
-            return AnthropicMessagesImageParam(
-                type="image",
-                source=AnthropicContentParamSourceUrl(
-                    type="url",
-                    url=image_url,
-                ),
-            )
+            return _anthropic_https_url_block(image_url, image_format)
     else:
         # Convert to base64 for data URIs or other formats
-        image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
-        return _anthropic_content_element_factory(image_chunk)
+        image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=image_format)
+        return anthropic_content_element_factory(image_chunk)
 
 
 # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@@ -1329,7 +1329,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
 _get_thought_signature_from_tool = get_thought_signature_from_tool
 
 
-def _get_dummy_thought_signature() -> str:
+def get_dummy_thought_signature() -> str:
     """Generate a dummy thought signature for models that require it.
 
     This is used when transferring conversation history from older models
@@ -1416,7 +1416,7 @@ def convert_to_gemini_tool_call_invoke(
                         # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
                         is_first_function_call = len(_parts_list) == 0
                         if not thought_signature and is_first_function_call and needs_dummy_signature:
-                            thought_signature = _get_dummy_thought_signature()
+                            thought_signature = get_dummy_thought_signature()
                         if thought_signature:
                             part_dict["thoughtSignature"] = thought_signature
 
@@ -1440,7 +1440,7 @@ def convert_to_gemini_tool_call_invoke(
 
                 # If no signature found and model is gemini-3, use dummy signature
                 if not thought_signature and needs_dummy_signature:
-                    thought_signature = _get_dummy_thought_signature()
+                    thought_signature = get_dummy_thought_signature()
 
                 if thought_signature:
                     part_dict_function["thoughtSignature"] = thought_signature
@@ -2255,16 +2255,16 @@ def add_cache_control_to_content(
     return anthropic_content_element
 
 
-def _anthropic_content_element_factory(
+def anthropic_content_element_factory(
     image_chunk: GenericImageParsingChunk,
 ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
-    if image_chunk["media_type"] == "application/pdf":
+    if is_pdf_media_type(image_chunk["media_type"]):
         _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = (
             AnthropicMessagesDocumentParam(
                 type="document",
                 source=AnthropicContentParamSource(
                     type="base64",
-                    media_type=image_chunk["media_type"],
+                    media_type="application/pdf",
                     data=image_chunk["data"],
                 ),
             )
@@ -4254,6 +4254,7 @@ def _convert_to_bedrock_tool_call_invoke(
     tool_calls: list,
     model: str | None = None,
     deployment_model: str | None = None,
+    model_info: Mapping[str, object] | None = None,
 ) -> list[BedrockContentBlock]:
     """
     OpenAI tool invokes:
@@ -4338,6 +4339,7 @@ def _convert_to_bedrock_tool_call_invoke(
                                     block_type="content_block",
                                     model=model,
                                     deployment_model=deployment_model,
+                                    model_info=model_info,
                                 )
                                 if _cache_point_block is not None:
                                     _parts_list.append(_cache_point_block)
@@ -4358,6 +4360,7 @@ def _convert_to_bedrock_tool_call_invoke(
                         block_type="content_block",
                         model=model,
                         deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     if cache_point_block is not None:
                         _parts_list.append(cache_point_block)
@@ -5029,6 +5032,7 @@ class BedrockConverseMessagesProcessor:
         deployment_model: str | None = None,
         base_model: str | None = None,
         drop_params: bool = False,
+        model_info: Mapping[str, object] | None = None,
     ) -> list[BedrockMessageBlock]:
         from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking
 
@@ -5091,6 +5095,7 @@ class BedrockConverseMessagesProcessor:
                                 block_type="content_block",
                                 model=model,
                                 deployment_model=deployment_model,
+                                model_info=model_info,
                             )
                             if _cache_point_block is not None:
                                 _parts.append(_cache_point_block)
@@ -5098,7 +5103,11 @@ class BedrockConverseMessagesProcessor:
                 elif message_content and isinstance(message_content, str):
                     _part = BedrockContentBlock(text=message_content)
                     _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
-                        message_block, block_type="content_block", model=model, deployment_model=deployment_model
+                        message_block,
+                        block_type="content_block",
+                        model=model,
+                        deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     user_content.append(_part)
                     if _cache_point_block is not None:
@@ -5149,6 +5158,7 @@ class BedrockConverseMessagesProcessor:
                         block_type="content_block",
                         model=model,
                         deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     if cache_point_block is not None:
                         tool_content.append(cache_point_block)
@@ -5214,6 +5224,7 @@ class BedrockConverseMessagesProcessor:
                             block_type="content_block",
                             model=model,
                             deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if _cache_point_block is not None and not (
                             drop_params and _as_thinking_block(element) is not None
@@ -5231,6 +5242,7 @@ class BedrockConverseMessagesProcessor:
                         block_type="content_block",
                         model=model,
                         deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     if _cache_point_block is not None:
                         assistant_content.append(_cache_point_block)
@@ -5239,7 +5251,7 @@ class BedrockConverseMessagesProcessor:
                 if _tool_calls:
                     assistant_content.extend(
                         _convert_to_bedrock_tool_call_invoke(
-                            _tool_calls, model=model, deployment_model=deployment_model
+                            _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info
                         )
                     )
 
@@ -5434,6 +5446,7 @@ def _bedrock_converse_messages_pt(
     deployment_model: str | None = None,
     base_model: str | None = None,
     drop_params: bool = False,
+    model_info: Mapping[str, object] | None = None,
 ) -> list[BedrockMessageBlock]:
     """
     Converts given messages from OpenAI format to Bedrock format
@@ -5502,6 +5515,7 @@ def _bedrock_converse_messages_pt(
                             block_type="content_block",
                             model=model,
                             deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if _cache_point_block is not None:
                             _parts.append(_cache_point_block)
@@ -5509,7 +5523,11 @@ def _bedrock_converse_messages_pt(
             elif message_content and isinstance(message_content, str):
                 _part = BedrockContentBlock(text=message_content)
                 _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
-                    message_block, block_type="content_block", model=model, deployment_model=deployment_model
+                    message_block,
+                    block_type="content_block",
+                    model=model,
+                    deployment_model=deployment_model,
+                    model_info=model_info,
                 )
                 user_content.append(_part)
                 if _cache_point_block is not None:
@@ -5562,6 +5580,7 @@ def _bedrock_converse_messages_pt(
                     block_type="content_block",
                     model=model,
                     deployment_model=deployment_model,
+                    model_info=model_info,
                 )
                 if cache_point_block is not None:
                     tool_content.append(cache_point_block)
@@ -5628,6 +5647,7 @@ def _bedrock_converse_messages_pt(
                             block_type="content_block",
                             model=model,
                             deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if _cache_point_block is not None and not (
                             drop_params and _as_thinking_block(element) is not None
@@ -5640,14 +5660,20 @@ def _bedrock_converse_messages_pt(
                     assistant_content.append(BedrockContentBlock(text=_assistant_content))
                 # Add cache point block for assistant string content
                 _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
-                    assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model
+                    assistant_message_block,
+                    block_type="content_block",
+                    model=model,
+                    deployment_model=deployment_model,
+                    model_info=model_info,
                 )
                 if _cache_point_block is not None:
                     assistant_content.append(_cache_point_block)
             _tool_calls = assistant_message_block.get("tool_calls", [])
             if _tool_calls:
                 assistant_content.extend(
-                    _convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model)
+                    _convert_to_bedrock_tool_call_invoke(
+                        _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info
+                    )
                 )
 
             msg_i += 1
@@ -5703,7 +5729,10 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
 
 
 def add_cache_point_tool_block(
-    tool: dict, model: str | None = None, deployment_model: str | None = None
+    tool: dict,
+    model: str | None = None,
+    deployment_model: str | None = None,
+    model_info: Mapping[str, object] | None = None,
 ) -> BedrockToolBlock | None:
     from litellm.llms.bedrock.common_utils import (
         bedrock_cache_point_ttl_supported,
@@ -5711,7 +5740,7 @@ def add_cache_point_tool_block(
     )
 
     cache_control: Final = tool.get("cache_control", None)
-    if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model):
+    if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model, model_info):
         cache_point: Final = cache_control.get("type", "ephemeral")
         if cache_point == "ephemeral":
             cache_point_block: Final[CachePointBlock] = {"type": "default"}
@@ -5748,7 +5777,10 @@ def _is_bedrock_tool_block(tool: dict) -> bool:
 
 
 def _bedrock_tools_pt(
-    tools: list, model: str | None = None, deployment_model: str | None = None
+    tools: list,
+    model: str | None = None,
+    deployment_model: str | None = None,
+    model_info: Mapping[str, object] | None = None,
 ) -> list[BedrockToolBlock]:
     """
     OpenAI tools looks like:
@@ -5869,7 +5901,9 @@ def _bedrock_tools_pt(
         tool_block_list.append(tool_block)
 
         ## ADD CACHE POINT TOOL BLOCK ##
-        cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model)
+        cache_point_tool_block = add_cache_point_tool_block(
+            tool, model=model, deployment_model=deployment_model, model_info=model_info
+        )
         if cache_point_tool_block is not None:
             tool_block_list.append(cache_point_tool_block)
 

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -5,28 +5,33 @@ Helper functions to handle images passed in messages
 import asyncio
 import base64
 import time
-from collections.abc import Callable, Iterable, Mapping
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
-from itertools import chain
+from functools import partial
+from itertools import accumulate, chain, takewhile, tee
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeVar
+from urllib.parse import urlparse
 
 import httpx
-from httpx import Response
+from httpx import USE_CLIENT_DEFAULT, Response
 
 import litellm
 from litellm import verbose_logger
 from litellm.caching.caching import InMemoryCache
-from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
+from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT
 from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
 from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
 from litellm.types.llms.openai import AllMessageValues
 
 MAX_IMGS_IN_MEMORY: Final = 10
 MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20
+_Fetched = TypeVar("_Fetched")
 _FETCH_ATTEMPTS: Final = 3
 
 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
+pdf_probe_verdicts: Final = InMemoryCache()
 
 
 def _header_value(response: Response, name: str) -> str | None:
@@ -209,6 +214,173 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool:
     return media.part_type == "image_url"
 
 
+_PDF_MEDIA_TYPES: Final = frozenset({"application/pdf", "application/x-pdf"})
+_GENERIC_MEDIA_TYPES: Final = frozenset({"application/octet-stream", "binary/octet-stream"})
+_PDF_SIGNATURE: Final = b"%PDF-"
+_PDF_SIGNATURE_WINDOW: Final = 1024
+_PDF_PROBE_HEADERS: Final = MappingProxyType({"Range": f"bytes=0-{_PDF_SIGNATURE_WINDOW - 1}"})
+_PDF_PROBE_STATUSES: Final = frozenset({200, 206})
+_PDF_PROBE_FAILURES: Final = (httpx.HTTPError, httpx.InvalidURL, ValueError)
+_ASYNC_PDF_PROBE_FAILURES: Final = (*_PDF_PROBE_FAILURES, asyncio.TimeoutError)
+_PDF_PROBE_VERDICT_TTL: Final = 600
+_PDF_PROBE_FAILURE_TTL: Final = 60
+_DECODED_BODY_HEADERS: Final = frozenset({"content-encoding", "content-length", "transfer-encoding"})
+
+
+def _bare_media_type(media_type: str) -> str:
+    return media_type.split(";", 1)[0].strip().lower()
+
+
+def is_pdf_media_type(media_type: str) -> bool:
+    return _bare_media_type(media_type) in _PDF_MEDIA_TYPES
+
+
+def is_unformatted_pdf_link(url: str, image_format: str | None) -> bool:
+    lowered: Final = url.lower()
+    return not image_format and (urlparse(lowered).path.endswith(".pdf") or lowered.endswith(".pdf"))
+
+
+def is_unformatted_pdf_link_image(media: RemoteMedia) -> bool:
+    image_format: Final = media.fields.get("format")
+    return inline_remote_image_urls(media) and is_unformatted_pdf_link(
+        media.url, image_format if isinstance(image_format, str) else None
+    )
+
+
+def _serves_pdf(response: Response) -> bool:
+    if response.status_code not in _PDF_PROBE_STATUSES:
+        return False
+    served_type: Final = _bare_media_type(_header_value(response, "Content-Type") or "")
+    if served_type and served_type not in _GENERIC_MEDIA_TYPES:
+        return served_type in _PDF_MEDIA_TYPES
+    return _PDF_SIGNATURE in response.content[:_PDF_SIGNATURE_WINDOW]
+
+
+def _signature_window_chunks(chunks: Iterable[bytes], deadline: float) -> Iterator[bytes]:
+    reading, counting = tee(chunks)
+    read_so_far: Final = chain((0,), accumulate(map(len, counting)))
+    still_reading: Final = takewhile(
+        lambda total: total < _PDF_SIGNATURE_WINDOW and time.monotonic() < deadline, read_so_far
+    )
+    return (chunk for _, chunk in zip(still_reading, reading))
+
+
+def _with_body_prefix(response: Response, prefix: bytes) -> Response:
+    return Response(
+        response.status_code,
+        headers=[(name, value) for name, value in response.headers.multi_items() if name not in _DECODED_BODY_HEADERS],
+        content=prefix,
+        request=response.request,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _BodyPrefixReader:
+    client: httpx.Client
+    deadline: float
+
+    def get(
+        self,
+        url: str,
+        *,
+        headers: Mapping[str, str] | None = None,
+        follow_redirects: bool = False,
+        timeout: float | httpx.Timeout | None = None,
+    ) -> Response:
+        with self.client.stream(
+            "GET",
+            url,
+            headers=headers,
+            follow_redirects=follow_redirects,
+            timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
+        ) as response:
+            prefix: Final = b"".join(_signature_window_chunks(response.iter_bytes(), self.deadline))
+            return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW])
+
+
+@dataclass(frozen=True, slots=True)
+class _AsyncBodyPrefixReader:
+    client: httpx.AsyncClient
+
+    async def get(
+        self,
+        url: str,
+        *,
+        headers: Mapping[str, str] | None = None,
+        follow_redirects: bool = False,
+        timeout: float | httpx.Timeout | None = None,
+        max_response_bytes: int | None = None,
+    ) -> Response:
+        async with self.client.stream(
+            "GET",
+            url,
+            headers=headers,
+            follow_redirects=follow_redirects,
+            timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
+        ) as response:
+            return _with_body_prefix(response, await anext(response.aiter_bytes(max_response_bytes), b""))
+
+
+def _cached_pdf_verdict(url: str) -> bool | None:
+    cached: Final = pdf_probe_verdicts.get_cache(url)
+    return cached if isinstance(cached, bool) else None
+
+
+def _remember_pdf_verdict(url: str, verdict: bool, ttl: int) -> bool:
+    pdf_probe_verdicts.set_cache(url, verdict, ttl=ttl)
+    return verdict
+
+
+def _remember_probe_response(url: str, response: Response) -> bool:
+    ttl: Final = _PDF_PROBE_VERDICT_TTL if response.status_code in _PDF_PROBE_STATUSES else _PDF_PROBE_FAILURE_TTL
+    return _remember_pdf_verdict(url, _serves_pdf(response), ttl)
+
+
+def _failed_pdf_probe(url: str, error: Exception) -> bool:
+    verbose_logger.debug("PDF probe of %s failed, sending it as an image link: %s", url, error)
+    return _remember_pdf_verdict(url, False, _PDF_PROBE_FAILURE_TTL)
+
+
+def serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
+    if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
+        return False
+    cached: Final = _cached_pdf_verdict(url)
+    if cached is not None:
+        return cached
+    try:
+        response: Final = safe_get(
+            _BodyPrefixReader(litellm.module_level_client.client, deadline=time.monotonic() + timeout),
+            url,
+            headers=dict(_PDF_PROBE_HEADERS),
+            timeout=timeout,
+        )
+    except _PDF_PROBE_FAILURES as e:
+        return _failed_pdf_probe(url, e)
+    return _remember_probe_response(url, response)
+
+
+async def async_serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
+    if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
+        return False
+    cached: Final = _cached_pdf_verdict(url)
+    if cached is not None:
+        return cached
+    try:
+        response: Final = await asyncio.wait_for(
+            async_safe_get(
+                _AsyncBodyPrefixReader(litellm.module_level_aclient.client),
+                url,
+                headers=dict(_PDF_PROBE_HEADERS),
+                timeout=timeout,
+                max_response_bytes=_PDF_SIGNATURE_WINDOW,
+            ),
+            timeout,
+        )
+    except _ASYNC_PDF_PROBE_FAILURES as e:
+        return _failed_pdf_probe(url, e)
+    return _remember_probe_response(url, response)
+
+
 def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None:
     if fields.get("type") != "image_url":
         return None
@@ -292,15 +464,40 @@ def _inline_part(part: object, data_urls: Mapping[str, str], should_inline: Call
     return _inline(remote, data_url) if data_url is not None else part
 
 
-def _inline_message(
-    message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
-) -> AllMessageValues:
+def _with_parts(message: AllMessageValues, rewrite: Callable[[object], object]) -> AllMessageValues:
     parts: Final = _content_parts(message)
     if not parts:
         return message
-    inlined_parts: Final = [_inline_part(part, data_urls, should_inline) for part in parts]
-    inlined_message: Final = {**message, "content": inlined_parts}
-    return inlined_message  # pyright: ignore[reportReturnType]  # the same message with its remote parts inlined
+    rewritten: Final = {**message, "content": [rewrite(part) for part in parts]}
+    return rewritten  # pyright: ignore[reportReturnType]  # the same message with some of its parts rewritten
+
+
+def _inline_message(
+    message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
+) -> AllMessageValues:
+    return _with_parts(message, partial(_inline_part, data_urls=data_urls, should_inline=should_inline))
+
+
+def _pdf_formatted_part(part: object, pdf_links: frozenset[str], should_probe: Callable[[RemoteMedia], bool]) -> object:
+    remote: Final = _parse_remote_part(part)
+    match remote:
+        case _RemoteImage(fields, image_url, url) if url in pdf_links and should_probe(_remote_media(remote)):
+            image_url_fields: Final = image_url if image_url is not None else _NO_FIELDS
+            return {**fields, "image_url": {**image_url_fields, "url": url, "format": "application/pdf"}}
+        case _:
+            return part
+
+
+def _with_pdf_formats(
+    messages: Iterable[AllMessageValues],
+    probed: Iterable[tuple[str, bool]],
+    should_probe: Callable[[RemoteMedia], bool],
+) -> tuple[AllMessageValues, ...]:
+    pdf_links: Final = frozenset(url for url, is_pdf in probed if is_pdf)
+    if not pdf_links:
+        return tuple(messages)
+    rewrite: Final = partial(_pdf_formatted_part, pdf_links=pdf_links, should_probe=should_probe)
+    return tuple(_with_parts(message, rewrite) for message in messages)
 
 
 async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str:
@@ -308,11 +505,12 @@ async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float
         return await async_convert_url_to_base64(url, timeout=timeout)
 
 
-async def async_convert_urls_to_base64(
-    remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
-) -> tuple[str, ...]:
-    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
-    fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
+async def _serves_pdf_within(url: str, in_flight: asyncio.Semaphore) -> bool:
+    async with in_flight:
+        return await async_serves_pdf(url)
+
+
+async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -> tuple[_Fetched, ...]:
     try:
         return tuple(await asyncio.gather(*fetches))
     except BaseException:
@@ -322,6 +520,15 @@ async def async_convert_urls_to_base64(
         raise
 
 
+async def async_convert_urls_to_base64(
+    remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
+) -> tuple[str, ...]:
+    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
+    return await _gather_or_cancel_all(
+        tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
+    )
+
+
 def _remote_urls_to_inline(
     messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
 ) -> tuple[str, ...]:
@@ -351,3 +558,27 @@ async def async_inline_remote_media(
     data_urls: Final = await async_convert_urls_to_base64(remote_urls)
     inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))
     return [_inline_message(message, inlined, should_inline) for message in messages]
+
+
+def format_pdf_links(
+    messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
+) -> tuple[AllMessageValues, ...]:
+    links: Final = _remote_urls_to_inline(messages, should_probe)
+    if not links:
+        return tuple(messages)
+    with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, len(links))) as probes:
+        verdicts: Final = tuple(probes.map(serves_pdf, links))
+    return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)
+
+
+async def async_format_pdf_links(
+    messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
+) -> tuple[AllMessageValues, ...]:
+    links: Final = _remote_urls_to_inline(messages, should_probe)
+    if not links:
+        return tuple(messages)
+    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
+    verdicts: Final = await _gather_or_cancel_all(
+        tuple(asyncio.create_task(_serves_pdf_within(url, in_flight)) for url in links)
+    )
+    return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)

litellm/litellm_core_utils/redact_messages.py

@@ -171,6 +171,8 @@ def _redact_streaming_response(streaming_response):
             _redact_bedrock_guardrail_trace(streaming_response)
     elif isinstance(streaming_response, litellm.ResponsesAPIResponse):
         return perform_redaction({}, streaming_response)
+    elif isinstance(streaming_response, litellm.TranscriptionResponse):
+        streaming_response.text = REDACTED_BY_LITELLM
     elif hasattr(streaming_response, "output"):
         _redact_responses_api_output(streaming_response.output)
         if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None:

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -34,6 +34,7 @@ from litellm.llms.anthropic.messages.adapters.transformation import (
 )
 from litellm.llms.anthropic.messages.convertible_blocks import (
     keep_non_string_tool_result_texts,
+    keep_url_documents,
     scannable_tool_result_request,
 )
 from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
@@ -1187,7 +1188,10 @@ class AnthropicMessagesHandler(BaseTranslation):
             cast(Sequence[object], converted),  # cast-ok: Anthropic message payload
             original_tool_names,
         )
-        data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages"))
+        original_messages: Final = data.get("messages")
+        data["messages"] = keep_url_documents(
+            keep_non_string_tool_result_texts(converted, original_messages), original_messages
+        )
 
     @staticmethod
     def _extract_midturn_system_text(

litellm/llms/anthropic/chat/handler.py

@@ -516,6 +516,7 @@ class AnthropicChatCompletion(BaseLLM):
                 headers=headers,
                 request_data=request_data,
                 provider=custom_llm_provider,
+                api_base=api_base,
             )
             data: Final = dict(
                 strip_beta_gated_request_body(
@@ -1194,6 +1195,9 @@ class ModelResponseIterator:
                 start_message: Final = message_start_block["message"]
                 if "model" in start_message:
                     self.served_model = start_message["model"]
+                input_transformations: Final = start_message.get("input_transformations")
+                if input_transformations is not None:
+                    provider_specific_fields["input_transformations"] = input_transformations
                 if "usage" in message_start_block["message"]:
                     usage = self._handle_usage(anthropic_usage_chunk=message_start_block["message"]["usage"])
             elif type_chunk == "error":

litellm/llms/anthropic/chat/transformation.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
     RESPONSE_FORMAT_TOOL_NAME,
 )
+from litellm.litellm_core_utils.asyncify import is_event_loop_running
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     DEFS_MAX_INLINED_BYTES,
@@ -47,8 +48,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     RemoteMedia,
+    async_format_pdf_links,
     async_inline_remote_media,
+    format_pdf_links,
     inline_remote_image_urls,
+    is_unformatted_pdf_link_image,
 )
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
     message_field,
@@ -2333,6 +2337,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
     def inlines_remote_media(self, media: RemoteMedia) -> bool:
         return inline_remote_image_urls(media) and media.url.startswith("http://")
 
+    def probes_pdf_link(self, media: RemoteMedia) -> bool:
+        return is_unformatted_pdf_link_image(media) and not self.inlines_remote_media(media)
+
+    def _with_pdf_links_formatted(self, messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]:
+        return messages if is_event_loop_running() else format_pdf_links(messages, should_probe=self.probes_pdf_link)
+
     async def async_transform_request(
         self,
         model: str,
@@ -2341,9 +2351,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         litellm_params: dict[str, object],  # mutable-ok: BaseConfig signature
         headers: dict[str, object],  # mutable-ok: BaseConfig signature
     ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+        inlined: Final = await async_inline_remote_media(messages, should_inline=self.inlines_remote_media)
         return self.transform_request(
             model=model,
-            messages=await async_inline_remote_media(messages, should_inline=self.inlines_remote_media),
+            messages=list(await async_format_pdf_links(inlined, should_probe=self.probes_pdf_link)),
             optional_params=optional_params,
             litellm_params=litellm_params,
             headers=headers,
@@ -2493,7 +2504,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         try:
             anthropic_messages = anthropic_messages_pt(
                 model=model,
-                messages=list(conversation),
+                messages=list(self._with_pdf_links_formatted(conversation)),
                 llm_provider=self._resolved_provider,
                 drop_params=drop_params,
             )
@@ -3058,7 +3069,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def _build_provider_specific_fields(
         self,
-        completion_response: dict,
+        completion_response: Mapping[str, object],
         citations: Sequence[object] | None,
         thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None,
         web_search_results: Sequence[object] | None,
@@ -3075,6 +3086,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if context_management is not None:
             provider_specific_fields["context_management"] = context_management
 
+        input_transformations: Final = completion_response.get("input_transformations")
+        if input_transformations is not None:
+            provider_specific_fields["input_transformations"] = input_transformations
+
         if web_search_results is not None:
             provider_specific_fields["web_search_results"] = web_search_results
 
@@ -3085,17 +3100,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if tool_results is not None:
             provider_specific_fields["tool_results"] = tool_results
 
+        container: Final = completion_response.get("container")
+        container_fields: Final[Mapping[str, object] | None] = container if isinstance(container, dict) else None
         if tool_results is not None and any(result.get("type") != "tool_search_tool_result" for result in tool_results):
-            container_id: Final = (
-                completion_response.get("container", {}).get("id")
-                if isinstance(completion_response.get("container"), dict)
-                else None
-            )
+            raw_container_id: Final = container_fields.get("id") if container_fields is not None else None
+            container_id: Final = raw_container_id if isinstance(raw_container_id, str) else None
             code_by_id: Final = self._build_code_by_id_map(tool_calls)
             code_interpreter_results = self._build_code_interpreter_results(tool_results, code_by_id, container_id)
             provider_specific_fields["code_interpreter_results"] = code_interpreter_results
 
-        container: Final = completion_response.get("container")
         if container is not None:
             provider_specific_fields["container"] = container
 
@@ -3196,8 +3209,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 STOP_REASON_FIELDS.validate_python(completion_response)
             )
             completion_response["stop_reason"] = "stop"
-            if stop_reason_fields:
-                json_mode_message.provider_specific_fields = dict(stop_reason_fields)
+            input_transformations: Final = provider_specific_fields.get("input_transformations")
+            json_mode_fields: Final = {
+                **stop_reason_fields,
+                **({} if input_transformations is None else {"input_transformations": input_transformations}),
+            }
+            if json_mode_fields:
+                json_mode_message.provider_specific_fields = json_mode_fields
             if "refusal" in refusal_fields:
                 json_mode_message.refusal = refusal_fields["refusal"]
             if thinking_blocks:

litellm/llms/anthropic/common_utils.py

@@ -28,7 +28,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
 )
 from litellm.exceptions import UnsupportedParamsError
-from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_file_ids_from_messages,
     is_encrypted_reasoning_block,
@@ -307,9 +307,10 @@ def count_tokens_feature_betas(
     )
 
 
-def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
+def supports_anthropic_cache_control(
+    model: str, custom_llm_provider: str | None, model_info: Mapping[str, object] | None = None
+) -> bool:
     from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
-    from litellm.utils import supports_prompt_caching
 
     try:
         provider: Final = custom_llm_provider if custom_llm_provider is not None else get_llm_provider(model=model)[1]
@@ -318,10 +319,27 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None
     return (
         provider in ("anthropic", "bedrock", "vertex_ai", "azure_ai")
         and "claude" in model.lower()
-        and supports_prompt_caching(model=model, custom_llm_provider=provider)
+        and _deployment_supports_prompt_caching(model, provider, model_info)
     )
 
 
+def _deployment_supports_prompt_caching(model: str, provider: str, model_info: Mapping[str, object] | None) -> bool:
+    from litellm.utils import catalog_with_runtime_registrations, get_model_info, supports_prompt_caching
+
+    declared: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching"))
+    if declared is not None:
+        return declared
+    live: Final = supports_prompt_caching(model=model, custom_llm_provider=provider)
+    if model_info is None:
+        return live
+    try:
+        key: Final = get_model_info(model=model, custom_llm_provider=provider).get("key")
+    except Exception:  # noqa: BLE001  # an unmapped model keeps the live answer
+        return live
+    catalog_entry: Final = catalog_with_runtime_registrations().get(key)
+    return live if catalog_entry is None else catalog_entry.get("supports_prompt_caching") is True
+
+
 def _tool_config_point_control(points: object) -> Mapping[object, object] | None:
     point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ())
     tool_config_point: Final = next(

litellm/llms/anthropic/count_tokens/transformation.py

@@ -49,7 +49,10 @@ _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"})
 def _count_image(block: JsonValue) -> JsonValue:
     if not isinstance(block, dict) or block.get("type") != "image_url":
         return block
-    from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        anthropic_content_element_factory,
+        convert_to_anthropic_image_obj,
+    )
 
     image_block: Final = _IMAGE_BLOCK.validate_python(block)
     image_url: Final = image_block["image_url"]
@@ -57,7 +60,7 @@ def _count_image(block: JsonValue) -> JsonValue:
         openai_image_url=image_url if isinstance(image_url, str) else image_url["url"],
         format=image_url.get("format") if isinstance(image_url, dict) else None,
     )
-    image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source})
+    image: Final = _COUNT_REQUEST.validate_python(anthropic_content_element_factory(source))
     return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image}
 
 

litellm/llms/anthropic/messages/convertible_blocks.py

@@ -314,6 +314,76 @@ def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: obj
     return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten]
 
 
+def _url_source_url(block: object) -> str | None:
+    match as_str_mapping(block):
+        case {"type": "image" | "document", "source": {"type": "url", "url": str(url)}}:
+            return url
+        case _:
+            return None
+
+
+def _is_document(block: object) -> bool:
+    match as_str_mapping(block):
+        case {"type": "document"}:
+            return True
+        case _:
+            return False
+
+
+@dataclass(frozen=True, slots=True, order=True)
+class _UrlBlockPlace:
+    in_tool_result: bool
+    url: str
+
+
+def _url_sourced_blocks(block: object, in_tool_result: bool = False) -> tuple[tuple[_UrlBlockPlace, object], ...]:
+    url: Final = _url_source_url(block)
+    if url is not None:
+        return ((_UrlBlockPlace(in_tool_result, url), block),)
+    match as_str_mapping(block):
+        case {"type": "tool_result", "content": list() as content}:
+            in_this_tool_result: Final = partial(_url_sourced_blocks, in_tool_result=True)
+            return tuple(chain.from_iterable(map(in_this_tool_result, _list_items(content))))
+        case _:
+            return ()
+
+
+def _entry_place(entry: tuple[_UrlBlockPlace, object]) -> _UrlBlockPlace:
+    return entry[0]
+
+
+def _entry_block(entry: tuple[_UrlBlockPlace, object]) -> object:
+    return entry[1]
+
+
+def _restored_url_document(
+    block: object, originals: Mapping[_UrlBlockPlace, Iterator[object]], in_tool_result: bool = False
+) -> object:
+    url: Final = _url_source_url(block)
+    if url is not None:
+        remaining: Final = originals.get(_UrlBlockPlace(in_tool_result, url))
+        original: Final = None if remaining is None else next(remaining, None)
+        return original if _is_document(original) else block
+    match as_str_mapping(block):
+        case {"type": "tool_result", "content": list() as content} as tool_result:
+            restore: Final = partial(_restored_url_document, originals=originals, in_tool_result=True)
+            return {**tool_result, "content": list(map(restore, _list_items(content)))}
+        case _:
+            return block
+
+
+def keep_url_documents(rewritten: Sequence[object], original: object) -> Sequence[object]:
+    blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original)))
+    by_place: Final = tuple(sorted(chain.from_iterable(map(_url_sourced_blocks, blocks)), key=_entry_place))
+    if not any(_is_document(block) for _, block in by_place):
+        return rewritten
+    originals_in_order: Final = MappingProxyType(
+        {place: iter(tuple(map(_entry_block, group))) for place, group in groupby(by_place, key=_entry_place)}
+    )
+    restore: Final = partial(_restored_url_document, originals=originals_in_order)
+    return [_with_user_blocks(message, restore) for message in rewritten]
+
+
 def _without_non_string_text(block: object) -> object:
     match as_str_mapping(block):
         case {"type": "tool_result", "content": list()} as tool_result:

litellm/llms/anthropic/messages/responses_adapters/handler.py

@@ -13,6 +13,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
+    routed_model_info,
     supports_openai_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.asyncify import (
@@ -187,6 +188,7 @@ def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, ob
         provider if isinstance(provider, str) else None,
         forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
         forwarded_kwargs.get("prompt_cache_options"),
+        routed_model_info(forwarded_kwargs.get("model_info")),
     )
 
 
@@ -194,10 +196,11 @@ def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[s
     provider: Final = forwarded_kwargs.get("custom_llm_provider")
     provider_name: Final = provider if isinstance(provider, str) else None
     base_model: Final = forwarded_kwargs.get("base_model")
-    return supports_openai_prompt_cache_breakpoint(model, provider_name) or (
+    model_info: Final = routed_model_info(forwarded_kwargs.get("model_info"))
+    return supports_openai_prompt_cache_breakpoint(model, provider_name, model_info) or (
         isinstance(base_model, str)
         and bool(base_model)
-        and supports_openai_prompt_cache_breakpoint(base_model, provider_name)
+        and supports_openai_prompt_cache_breakpoint(base_model, provider_name, model_info)
     )
 
 

litellm/llms/azure/audio_transcriptions.py

@@ -1,11 +1,17 @@
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import TYPE_CHECKING, Final
 
-from openai import AsyncAzureOpenAI, AzureOpenAI
+from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream
+from openai.types.audio import TranscriptionStreamEvent
 from pydantic import BaseModel
 
 from litellm._uuid import uuid
 from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    acreate_sdk_transcription,
+    create_sdk_transcription,
+    sdk_compatible_transcription_request_data,
+)
 from litellm.types.utils import FileTypes
 from litellm.utils import (
     TranscriptionResponse,
@@ -38,13 +44,18 @@ class AzureAudioTranscription(AzureChatCompletion):
         atranscription: bool = False,
         litellm_params: dict | None = None,
         custom_llm_provider: str = "azure",
-    ) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
+    ) -> (
+        TranscriptionResponse
+        | Stream[TranscriptionStreamEvent]
+        | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
+    ):
         data: Final = {"model": model, "file": audio_file, **optional_params}
+        sdk_data: Final = sdk_compatible_transcription_request_data(data)
 
         if atranscription is True:
             return self.async_audio_transcriptions(
                 audio_file=audio_file,
-                data=data,
+                data=sdk_data,
                 model_response=model_response,
                 timeout=timeout,
                 api_key=api_key,
@@ -58,7 +69,7 @@ class AzureAudioTranscription(AzureChatCompletion):
             )
 
         azure_client: Final = self.get_azure_openai_client(
-            api_version=api_version,
+            api_version=None if self.is_azure_v1_api_version(api_version) else api_version,
             api_base=api_base,
             api_key=api_key,
             model=model,
@@ -84,10 +95,9 @@ class AzureAudioTranscription(AzureChatCompletion):
             },
         )
 
-        response: Final = azure_client.audio.transcriptions.create(
-            **data,
-            timeout=timeout,
-        )
+        _, response = create_sdk_transcription(azure_client, sdk_data, timeout)
+        if isinstance(response, Stream):
+            return response
 
         if isinstance(response, BaseModel):
             stringified_response = response.model_dump()
@@ -114,7 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion):
         self,
         audio_file: FileTypes,
         model: str,
-        data: dict,
+        data: Mapping[str, object],
         model_response: TranscriptionResponse,
         timeout: float,
         logging_obj: "LiteLLMLoggingObj",
@@ -125,7 +135,7 @@ class AzureAudioTranscription(AzureChatCompletion):
         max_retries=None,
         litellm_params: dict | None = None,
         custom_llm_provider: str = "azure",
-    ) -> TranscriptionResponse:
+    ) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]:
         response = None
         try:
             async_azure_client: Final = self.get_azure_openai_client(
@@ -155,12 +165,9 @@ class AzureAudioTranscription(AzureChatCompletion):
                 },
             )
 
-            raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create(
-                **data, timeout=timeout
-            )
-
-            headers: Final = dict(raw_response.headers)
-            response = raw_response.parse()
+            headers, response = await acreate_sdk_transcription(async_azure_client, data, timeout)
+            if isinstance(response, AsyncStream):
+                return response
 
             if isinstance(response, BaseModel):
                 stringified_response = response.model_dump()

litellm/llms/azure/chat/gpt_transformation.py

@@ -1,11 +1,17 @@
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    cast,  # noqa: TID251  # non_default_params reaches the mapper as an untyped dict
+)
 
 from httpx._models import Headers, Response
 
 import litellm
 from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support
+from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     chat_completion_prompt_cache_breakpoints_for_wire,
@@ -245,12 +251,12 @@ class AzureOpenAIConfig(BaseConfig):
                         or (api_version_year == "2023" and api_version_month < "12")
                         or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01")
                     ):
-                        if litellm.drop_params is True or (drop_params is not None and drop_params is True):
-                            pass
-                        else:
+                        if is_allowed_tools_tool_choice(
+                            cast(object, value)  # cast-ok: widening an untyped dict value to object is always sound
+                        ) or not (litellm.drop_params is True or (drop_params is not None and drop_params is True)):
                             raise UnsupportedParamsError(
                                 status_code=400,
-                                message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""",
+                                message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. drop_params never drops an allowed_tools tool_choice, since that would let the model call tools the request excluded. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""",
                             )
                     elif value == "required" and (
                         api_version_year == "2024" and api_version_month <= "05"

litellm/llms/azure/realtime/handler.py

@@ -5,9 +5,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
 """
 
 from collections.abc import Mapping
+from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final, Protocol, cast
 
+import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.litellm_core_utils.url_utils import join_api_base
@@ -48,17 +50,55 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe
         pass
 
 
+def azure_realtime_requires_ga(model: str) -> bool:
+    try:
+        azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure")
+    except Exception:  # noqa: BLE001  # unmapped deployments can select a protocol explicitly
+        try:
+            openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai")
+        except Exception:  # noqa: BLE001  # unmapped deployments can select a protocol explicitly
+            return False
+        openai_entry: Final = openai_model_info.get("provider_specific_entry")
+        return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1
+    azure_entry: Final = azure_model_info.get("provider_specific_entry")
+    return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1
+
+
+def _is_ga_protocol(protocol: str | None) -> bool:
+    return protocol is not None and protocol.upper() in ("GA", "V1")
+
+
+@dataclass(frozen=True, slots=True)
+class AzureRealtimeProtocol:
+    name: str
+    bridges_beta_client_to_ga: bool = False
+
+
 def azure_realtime_protocol_for_client(
     configured_protocol: object,
     *,
+    model: str,
+    base_model: str | None = None,
+    realtime_mode: str,
     query_params: RealtimeQueryParams | None,
     websocket: ScopedWebSocket,
-) -> str:
+) -> AzureRealtimeProtocol:
+    serves_a_ga_only_session: Final = (
+        realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription"
+    )
+    client_wants_beta: Final = client_sent_openai_beta_realtime_header(websocket)
     if isinstance(configured_protocol, str) and configured_protocol:
-        return configured_protocol
-    if (query_params or {}).get("intent") == "transcription":
-        return "GA"
-    return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA"
+        return AzureRealtimeProtocol(
+            name=configured_protocol,
+            bridges_beta_client_to_ga=client_wants_beta
+            and not serves_a_ga_only_session
+            and _is_ga_protocol(configured_protocol),
+        )
+    if serves_a_ga_only_session:
+        return AzureRealtimeProtocol(name="GA")
+    if azure_realtime_requires_ga(base_model or model):
+        return AzureRealtimeProtocol(name="GA", bridges_beta_client_to_ga=client_wants_beta)
+    return AzureRealtimeProtocol(name="beta" if client_wants_beta else "GA")
 
 
 class _ProxyClientWebSocket(Protocol):
@@ -92,6 +132,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
         api_version: str | None,
         realtime_protocol: str | None = None,
         query_params: RealtimeQueryParams | None = None,
+        realtime_mode: str = "realtime",
     ) -> str:
         """
         Construct Azure realtime WebSocket URL.
@@ -104,6 +145,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
                 - "GA" or "v1": Uses /openai/v1/realtime (GA path)
                 - "beta" or None: Uses /openai/realtime (beta path, default)
             query_params: Extra query params to forward (e.g. intent=transcription).
+            realtime_mode: Realtime surface to target, "realtime" or "translation".
 
         Returns:
             WebSocket URL string
@@ -117,24 +159,29 @@ class AzureOpenAIRealtime(AzureChatCompletion):
         api_base = api_base.replace("https://", "wss://")
 
         # Determine path based on realtime_protocol (case-insensitive)
-        _is_ga: Final = realtime_protocol is not None and realtime_protocol.upper() in (
-            "GA",
-            "V1",
-        )
+        _is_ga: Final = _is_ga_protocol(realtime_protocol)
         intent: Final = (query_params or {}).get("intent")
 
-        if _is_ga:
-            path = "/openai/v1/realtime"
-            query_parts = []
-            if intent != "transcription" and (query_params is None or "model" in query_params):
-                query_parts.append(urlencode({"model": model}))
-        else:
-            # Default to beta path for backwards compatibility
-            path = "/openai/realtime"
-            query_parts = [urlencode({"api-version": api_version, "deployment": model})]
+        path: Final = (
+            "/openai/v1/realtime/translations"
+            if realtime_mode == "translation"
+            else "/openai/v1/realtime"
+            if _is_ga
+            else "/openai/realtime"
+        )
+        base_query_parts: Final = (
+            (urlencode((("model", model),)),)
+            if realtime_mode == "translation"
+            else (
+                (urlencode((("model", model),)),)
+                if intent != "transcription" and (query_params is None or "model" in query_params)
+                else ()
+            )
+            if _is_ga
+            else (urlencode((("api-version", api_version), ("deployment", model))),)
+        )
 
-        if intent:
-            query_parts.append(urlencode({"intent": intent}))
+        query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts
 
         qs: Final = "&".join(query_parts)
         endpoint_url: Final = join_api_base(api_base, path)
@@ -165,13 +212,16 @@ class AzureOpenAIRealtime(AzureChatCompletion):
         query_params: RealtimeQueryParams | None = None,
         user_api_key_dict: object | None = None,
         litellm_metadata: dict | None = None,
+        realtime_mode: str = "realtime",
+        bridges_beta_client_to_ga: bool = False,
+        drop_params: bool = False,
     ):
         import websockets
         from websockets.asyncio.client import ClientConnection
 
         if api_base is None:
             raise ValueError("api_base is required for Azure OpenAI calls")
-        backend_uses_beta_protocol: Final = realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")
+        backend_uses_beta_protocol: Final = not _is_ga_protocol(realtime_protocol)
         if api_version is None and backend_uses_beta_protocol:
             raise ValueError("api_version is required for Azure OpenAI calls")
 
@@ -181,6 +231,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
             api_version,
             realtime_protocol=realtime_protocol,
             query_params=query_params,
+            realtime_mode=realtime_mode,
         )
 
         auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token)
@@ -204,6 +255,9 @@ class AzureOpenAIRealtime(AzureChatCompletion):
                     force_transcription_model=(
                         model if (query_params or {}).get("intent") == "transcription" else None
                     ),
+                    translation_session=realtime_mode == "translation",
+                    bridges_beta_client_to_ga=bridges_beta_client_to_ga,
+                    drop_params=drop_params,
                 )
                 await realtime_streaming.bidirectional_forward()
 

litellm/llms/azure/realtime/http_transformation.py

@@ -1,11 +1,20 @@
 """Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls)."""
 
+from collections.abc import Mapping
 from typing import Final
 
 import litellm
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.azure.common_utils import get_azure_ad_token
 from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
+
+_CALLER_AUTH_HEADERS: Final = frozenset({"authorization", "api-key"})
+
+
+def _azure_resource_base(api_base: str) -> str:
+    return api_base.rstrip("/").removesuffix("/openai/v1").removesuffix("/openai")
 
 
 def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
@@ -21,6 +30,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
     def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
         return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
 
+    def get_extra_headers(
+        self,
+        litellm_params: GenericLiteLLMParams,
+        api_key: str,
+        extra_headers: Mapping[str, object] | None,
+    ) -> Mapping[str, object] | None:
+        if api_key or any(name.lower() in _CALLER_AUTH_HEADERS for name in extra_headers or {}):
+            return extra_headers
+        azure_ad_token: Final = get_azure_ad_token(litellm_params)
+        if azure_ad_token is None:
+            return extra_headers
+        return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"}
+
     def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
         return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
 
@@ -30,20 +52,30 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
         model: str,
         api_key: str | None = None,
     ) -> dict:
-        return {
+        validated_headers: Final = {
             **headers,
-            "api-key": api_key or "",
             "Content-Type": "application/json",
         }
+        if api_key:
+            validated_headers["api-key"] = api_key
+        return validated_headers
 
     def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
         return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
 
     def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
-        base: Final = self.get_api_base(api_base)
+        base: Final = _azure_resource_base(self.get_api_base(api_base))
         version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
         return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}"
 
+    def get_translation_client_secret_url(
+        self, api_base: str | None, model: str, api_version: str | None = None
+    ) -> str:
+        return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/client_secrets")
+
+    def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
+        return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/calls")
+
     def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
         return {
             "Authorization": f"Bearer {ephemeral_key}",

litellm/llms/azure_ai/chat/transformation.py

@@ -10,7 +10,11 @@ from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+from litellm.integrations.anthropic_cache_control_hook import (
+    AnthropicCacheControlHook,
+    declared_prompt_cache_breakpoint,
+    routed_model_info,
+)
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     content_parts,
     convert_content_list_to_str,
@@ -125,18 +129,14 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool)
 
 
 def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
+    model_info: Final = routed_model_info(litellm_params.get("model_info"))
+    declared: Final = declared_prompt_cache_breakpoint(model_info)
+    if declared is not None:
+        return declared
     base_model: Final = litellm_params.get("base_model")
     named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
-    model_info: Final = litellm_params.get("model_info")
-    if not isinstance(model_info, Mapping):
-        return any(
-            AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
-        )
-    deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
-    if isinstance(deployment_flag, bool):
-        return deployment_flag
     return any(
-        AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
+        AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai", model_info=model_info)
         for name in named_models
     )
 

litellm/llms/base_llm/audio_transcription/transformation.py

@@ -1,9 +1,30 @@
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Final
+from types import MappingProxyType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    cast,  # noqa: TID251  # SDK file handles have no runtime validation schema
+)
 
 import httpx
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
+from openai.types.audio import (
+    Transcription,
+    TranscriptionDiarized,
+    TranscriptionStreamEvent,
+    TranscriptionVerbose,
+)
+from openai.types.audio.transcription_create_params import (
+    TranscriptionCreateParamsBase,
+    TranscriptionCreateParamsStreaming,
+)
+from pydantic import BaseModel, TypeAdapter
 
+from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -11,6 +32,31 @@ from litellm.types.llms.openai import (
 )
 from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
 
+_TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object])
+_TRANSCRIPTION_TEXT: Final = TypeAdapter(str)
+_TRANSCRIPTION_BODY: Final = TypeAdapter(dict[str, object])
+_TRANSCRIPTION_TYPES_BY_FORMAT: Final = MappingProxyType(
+    {"verbose_json": TranscriptionVerbose, "diarized_json": TranscriptionDiarized}
+)
+
+
+def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]:
+    request_data: Final = _TRANSCRIPTION_REQUEST.validate_python(data)
+    extension_keys: Final = ("keywords", "languages")
+    extension_body: Final = {key: request_data[key] for key in extension_keys if key in request_data}
+    if not extension_body:
+        return request_data
+
+    existing_extra_body: Final = request_data.get("extra_body")
+    return {
+        **{key: value for key, value in request_data.items() if key not in extension_keys},
+        "extra_body": {
+            **(as_str_mapping(existing_extra_body) or {}),
+            **extension_body,
+        },
+    }
+
+
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
@@ -20,6 +66,77 @@ else:
     LiteLLMLoggingObj = Any
 
 
+def sdk_compatible_transcription_response(response: object) -> BaseModel | str:
+    if isinstance(response, BaseModel):
+        return response
+    return _TRANSCRIPTION_TEXT.validate_python(response, strict=True)
+
+
+def _content_type(http_response: httpx.Response) -> str:
+    return next(iter(http_response.headers.get_list("content-type")), "")
+
+
+def _is_event_stream(http_response: httpx.Response) -> bool:
+    return _content_type(http_response).startswith("text/event-stream")
+
+
+def _unstreamed_transcription_response(http_response: httpx.Response, response_format: object) -> BaseModel | str:
+    if not _content_type(http_response).startswith("application/json"):
+        return http_response.text
+    transcription_type: Final = _TRANSCRIPTION_TYPES_BY_FORMAT.get(str(response_format), Transcription)
+    return transcription_type.model_construct(None, **_TRANSCRIPTION_BODY.validate_json(http_response.content))
+
+
+def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase:
+    return cast(  # cast-ok: SDK file handles have no runtime validation schema
+        TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"}
+    )
+
+
+def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsStreaming:
+    return cast(  # cast-ok: SDK file handles have no runtime validation schema
+        TranscriptionCreateParamsStreaming, data
+    )
+
+
+def create_sdk_transcription(
+    client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout
+) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]:
+    if not transcription_stream_requested(data.get("stream")):
+        raw_response: Final = client.audio.transcriptions.with_raw_response.create(
+            **_non_streaming_params(data), timeout=timeout
+        )
+        return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse())
+    raw_stream: Final = client.audio.transcriptions.with_raw_response.create(
+        **_streaming_params({**data, "stream": True}), timeout=timeout
+    )
+    if _is_event_stream(raw_stream.http_response):
+        return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent])
+    raw_stream.http_response.read()
+    return dict(raw_stream.headers), _unstreamed_transcription_response(
+        raw_stream.http_response, data.get("response_format")
+    )
+
+
+async def acreate_sdk_transcription(
+    client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout
+) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]:
+    if not transcription_stream_requested(data.get("stream")):
+        raw_response: Final = await client.audio.transcriptions.with_raw_response.create(
+            **_non_streaming_params(data), timeout=timeout
+        )
+        return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse())
+    raw_stream: Final = await client.audio.transcriptions.with_raw_response.create(
+        **_streaming_params({**data, "stream": True}), timeout=timeout
+    )
+    if _is_event_stream(raw_stream.http_response):
+        return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent])
+    await raw_stream.http_response.aread()
+    return dict(raw_stream.headers), _unstreamed_transcription_response(
+        raw_stream.http_response, data.get("response_format")
+    )
+
+
 @dataclass
 class AudioTranscriptionRequestData:
     """

litellm/llms/base_llm/decisions/transformation.py

@@ -431,6 +431,10 @@ class DecisionsResponseValidationError(BaseLLMException):
     pass
 
 
+def _lacks_confidence(answer: DecisionsIRAnswer) -> bool:
+    return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None
+
+
 def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
     return systemone_response_to_ir(
         _SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request
@@ -534,6 +538,8 @@ class BaseDecisionsConfig(ABC):
     max_total_image_bytes: int | None = None
     image_media_types: frozenset[str] | None = None
     inlines_remote_images: bool = False
+    openai_evidence_fields: frozenset[str] = frozenset()
+    confidence_optional: bool = False
     health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
         {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
     )
@@ -682,13 +688,20 @@ class BaseDecisionsConfig(ABC):
     ) -> DecisionsIRResponse:
         try:
             payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content)
-            return self.parse_response(payload, request)
+            response: Final = self.parse_response(payload, request)
         except ValidationError as error:
             raise DecisionsResponseValidationError(
                 status_code=500,
                 message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
                 f"{error.title} {_validation_summary(error)}",
             ) from error
+        if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers):
+            raise DecisionsResponseValidationError(
+                status_code=500,
+                message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
+                "a choice or score answer has no confidence",
+            )
+        return response
 
     def get_error_class(
         self,

litellm/llms/base_llm/google_genai/transformation.py

@@ -164,7 +164,7 @@ class BaseGoogleGenAIGenerateContentConfig(ABC):
         tools: ToolConfigDict | None,
         generate_content_config_dict: dict,
         system_instruction: object | None = None,
-    ) -> dict:
+    ) -> dict[str, object]:
         """
         Transform the request parameters for the generate content API.
 

litellm/llms/base_llm/realtime/http_transformation.py

@@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow:
 """
 
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from typing import Final
 
 import httpx
 
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.types.router import GenericLiteLLMParams
 
 
 class BaseRealtimeHTTPConfig(ABC):
@@ -50,6 +53,14 @@ class BaseRealtimeHTTPConfig(ABC):
           explicit api_key → litellm.api_key → env var → ""
         """
 
+    def get_extra_headers(
+        self,
+        litellm_params: GenericLiteLLMParams,
+        api_key: str,
+        extra_headers: Mapping[str, object] | None,
+    ) -> Mapping[str, object] | None:
+        return extra_headers
+
     # ------------------------------------------------------------------ #
     # client_secrets endpoint                                              #
     # ------------------------------------------------------------------ #
@@ -64,6 +75,12 @@ class BaseRealtimeHTTPConfig(ABC):
         base = base.removesuffix("/v1")
         return join_api_base(base, "v1/realtime/transcription_sessions")
 
+    def get_translation_client_secret_url(
+        self, api_base: str | None, model: str, api_version: str | None = None
+    ) -> str:
+        base: Final = (api_base or "").rstrip("/")
+        return f"{base}/v1/realtime/translations/client_secrets"
+
     @abstractmethod
     def validate_environment(
         self,
@@ -86,6 +103,10 @@ class BaseRealtimeHTTPConfig(ABC):
         """Return the full URL for POST /realtime/calls (SDP exchange)."""
         return join_api_base(api_base or "", "v1/realtime/calls")
 
+    def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
+        base: Final = (api_base or "").rstrip("/")
+        return f"{base}/v1/realtime/translations/calls"
+
     def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
         """
         Build headers for the realtime_calls POST.

litellm/llms/bedrock/chat/converse_transformation.py

@@ -69,6 +69,7 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
 from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
 from litellm.llms.bedrock.common_utils import (
+    bedrock_chat_opts_into_native_structured_output,
     bedrock_chat_uses_native_structured_output,
     bedrock_cleaned_output_schema,
     bedrock_model_is_unknown,
@@ -955,8 +956,9 @@ class AmazonConverseConfig(BaseConfig):
         """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
 
         ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
-        ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
-        request asks for thinking, which the json tool path would drop.
+        ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the
+        request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot
+        force the json tool gets it unless that key is false.
         Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
         """
         return bedrock_chat_uses_native_structured_output(
@@ -1337,19 +1339,15 @@ class AmazonConverseConfig(BaseConfig):
         if "type" in value and value["type"] == "text":
             return optional_params
 
+        thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params))
         native_schema: Final = (
-            self._native_output_schema(
-                model,
-                json_schema,
-                self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
-                model_info,
-            )
+            self._native_output_schema(model, json_schema, thinking_requested, model_info)
             if json_schema is not None
             else None
         )
         if native_schema is not None:
             # Use Bedrock's native structured outputs API (outputConfig.textFormat)
-            # No synthetic tool injection, no fake_stream needed.
+            # No synthetic tool injection.
             # Requires an explicit schema — json_object with no schema falls through
             # to the tool-call path below.
             output_config: Final = self._create_output_config_for_response_format(
@@ -1375,8 +1373,6 @@ class AmazonConverseConfig(BaseConfig):
                 optional_params["tool_choice"] = ToolChoiceValuesBlock(
                     tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
                 )
-            if non_default_params.get("stream", False) is True:
-                optional_params["fake_stream"] = True
         # else: response_format=json_object with no schema.
         # Don't inject the synthetic json_tool_call tool here. When no
         # schema is given, _create_json_tool_call_for_response_format
@@ -1384,6 +1380,13 @@ class AmazonConverseConfig(BaseConfig):
         # returns {} instead of the requested JSON. The model already
         # returns JSON when the prompt asks for it.
 
+        # A request that goes native only because the model can't force the json tool keeps the json tool's faked
+        # stream, so it still needs only bedrock:InvokeModel.
+        streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output(
+            model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info
+        )
+        if json_schema is not None and not streams_natively and non_default_params.get("stream", False) is True:
+            optional_params["fake_stream"] = True
         optional_params["json_mode"] = True
         return optional_params
 
@@ -1438,6 +1441,7 @@ class AmazonConverseConfig(BaseConfig):
         block_type: Literal["system"],
         model: str | None = None,
         deployment_model: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> SystemContentBlock | None:
         pass
 
@@ -1451,6 +1455,7 @@ class AmazonConverseConfig(BaseConfig):
         block_type: Literal["content_block"],
         model: str | None = None,
         deployment_model: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> ContentBlock | None:
         pass
 
@@ -1463,9 +1468,10 @@ class AmazonConverseConfig(BaseConfig):
         block_type: Literal["system", "content_block"],
         model: str | None = None,
         deployment_model: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> SystemContentBlock | ContentBlock | None:
         cache_control: Final = message_block.get("cache_control", None)
-        if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model):
+        if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model, model_info):
             return None
 
         cache_point: Final = self._build_cache_point_block(cache_control, model, deployment_model)
@@ -1618,8 +1624,9 @@ class AmazonConverseConfig(BaseConfig):
         drop_params: bool,
         optional_params: Mapping[str, object],
         litellm_params: Mapping[str, object],
+        model_info: Mapping[str, object] | None,
     ) -> Sequence[AllMessageValues]:
-        if not bedrock_model_known_to_accept_cache_points(model, deployment_model):
+        if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info):
             return messages
         markable: Final = (
             AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model)
@@ -1630,7 +1637,9 @@ class AmazonConverseConfig(BaseConfig):
             markable,
             drop_params,
             AnthropicCacheControlHook.count_converse_cache_points,
-            lambda: self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params),
+            lambda: self._tool_cache_points_on_the_wire(
+                model, deployment_model, optional_params, litellm_params, model_info
+            ),
         )
 
     def _tool_cache_points_on_the_wire(
@@ -1639,10 +1648,11 @@ class AmazonConverseConfig(BaseConfig):
         deployment_model: str,
         optional_params: Mapping[str, object],
         litellm_params: Mapping[str, object],
+        model_info: Mapping[str, object] | None,
     ) -> int:
         tools: Final = optional_params.get("tools")
         bedrock_tools, _ = self._process_tools_and_beta(
-            tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model
+            tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info
         )
         injects_tool_config_point: Final = bool(bedrock_tools) and any(
             point.get("location") == "tool_config"
@@ -1658,9 +1668,10 @@ class AmazonConverseConfig(BaseConfig):
         deployment_model: str,
         optional_params: Mapping[str, object],
         litellm_params: Mapping[str, object],
+        model_info: Mapping[str, object] | None,
     ) -> bool:
         if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points(
-            model, deployment_model
+            model, deployment_model, model_info
         ):
             return False
         message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages))
@@ -1669,12 +1680,16 @@ class AmazonConverseConfig(BaseConfig):
         cache_points: Final = (
             sum(1 for block in system_content_blocks if "cachePoint" in block)
             + message_cache_points
-            + self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params)
+            + self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info)
         )
         return cache_points <= MAX_CACHE_CONTROL_BLOCKS
 
     def _transform_system_message(
-        self, messages: list[AllMessageValues], model: str | None = None, deployment_model: str | None = None
+        self,
+        messages: list[AllMessageValues],
+        model: str | None = None,
+        deployment_model: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> tuple[list[AllMessageValues], list[SystemContentBlock]]:
         leading_count: Final = next(
             (i for i, m in enumerate(messages) if not is_system_message(m)),
@@ -1690,7 +1705,7 @@ class AmazonConverseConfig(BaseConfig):
             if isinstance(content, str) and content:
                 system_content_blocks.append(SystemContentBlock(text=content))
                 cache_block = self.get_cache_point_block(
-                    message, block_type="system", model=model, deployment_model=deployment_model
+                    message, block_type="system", model=model, deployment_model=deployment_model, model_info=model_info
                 )
                 if cache_block:
                     system_content_blocks.append(cache_block)
@@ -1699,7 +1714,11 @@ class AmazonConverseConfig(BaseConfig):
                     if m.get("type") == "text" and m.get("text"):
                         system_content_blocks.append(SystemContentBlock(text=m["text"]))
                         cache_block = self.get_cache_point_block(
-                            m, block_type="system", model=model, deployment_model=deployment_model
+                            m,
+                            block_type="system",
+                            model=model,
+                            deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if cache_block:
                             system_content_blocks.append(cache_block)
@@ -1897,6 +1916,7 @@ class AmazonConverseConfig(BaseConfig):
         additional_request_params: dict,
         litellm_params: Mapping[str, object] | None = None,
         deployment_model: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> tuple[list[ToolBlock], list]:
         """Process tools and collect anthropic_beta values."""
         bedrock_tools: list[ToolBlock] = []
@@ -1958,6 +1978,7 @@ class AmazonConverseConfig(BaseConfig):
                 _tools_the_model_accepts(regular_tools, model, litellm_params),
                 model=model,
                 deployment_model=deployment_model,
+                model_info=model_info,
             )
 
             # Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
@@ -2027,6 +2048,7 @@ class AmazonConverseConfig(BaseConfig):
                 _tools_the_model_accepts(_TOOLS_AS_SENT.validate_python(filtered_tools), model, litellm_params),
                 model=model,
                 deployment_model=deployment_model,
+                model_info=model_info,
             )
 
         # Append pre-formatted tools (systemTool etc.) after transformation
@@ -2102,6 +2124,7 @@ class AmazonConverseConfig(BaseConfig):
         drop_params: bool = False,
         litellm_params: Mapping[str, object] | None = None,
         deployment_model: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> CommonRequestObject:
         # Drop thinking param if thinking is enabled but thinking_blocks are missing
         # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
@@ -2140,7 +2163,7 @@ class AmazonConverseConfig(BaseConfig):
 
         # Process tools and collect beta values
         bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
-            original_tools, model, headers, additional_request_params, litellm_params, deployment_model
+            original_tools, model, headers, additional_request_params, litellm_params, deployment_model, model_info
         )
         requested_thinking: Final = additional_request_params.get("thinking")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # _prepare_request_params hands back a bare dict
         body_betas: Final = additional_request_params.get("anthropic_beta")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # same bare dict
@@ -2158,7 +2181,7 @@ class AmazonConverseConfig(BaseConfig):
         if (
             cache_injection_points
             and len(bedrock_tools) > 0
-            and bedrock_model_accepts_cache_points(model, deployment_model)
+            and bedrock_model_accepts_cache_points(model, deployment_model, model_info)
         ):
             for point in cache_injection_points:
                 if point.get("location") == "tool_config":
@@ -2244,8 +2267,11 @@ class AmazonConverseConfig(BaseConfig):
         litellm_params: dict[str, object],
         headers: dict | None = None,
     ) -> RequestObject:
+        from litellm.integrations.anthropic_cache_control_hook import routed_model_info
+
         base_model: Final = litellm_params.get("base_model")
         capability_model: Final = bedrock_capability_model(model, base_model)
+        model_info: Final = routed_model_info(litellm_params.get("model_info"))
         json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
         marked_messages: Final = self._messages_with_cache_marks(
             messages,
@@ -2254,6 +2280,7 @@ class AmazonConverseConfig(BaseConfig):
             drops_unsupported_params(litellm_params.get("drop_params")),
             json_optional_params,
             litellm_params,
+            model_info,
         )
         messages, system_content_blocks = self._transform_system_message(
             list(
@@ -2263,6 +2290,7 @@ class AmazonConverseConfig(BaseConfig):
             ),
             model=capability_model,
             deployment_model=model,
+            model_info=model_info,
         )
 
         # Convert last user message to guarded_text if guardrailConfig is present
@@ -2271,7 +2299,13 @@ class AmazonConverseConfig(BaseConfig):
             guard_input,
             optional_params,
             keeps_cache_marks=self._guarded_text_keeps_cache_marks(
-                guard_input, system_content_blocks, capability_model, model, json_optional_params, litellm_params
+                guard_input,
+                system_content_blocks,
+                capability_model,
+                model,
+                json_optional_params,
+                litellm_params,
+                model_info,
             ),
         )
         ## TRANSFORMATION ##
@@ -2279,6 +2313,7 @@ class AmazonConverseConfig(BaseConfig):
         _data: Final[CommonRequestObject] = self._transform_request_helper(
             model=capability_model,
             deployment_model=model,
+            model_info=model_info,
             system_content_blocks=system_content_blocks,
             optional_params=optional_params,
             messages=messages,
@@ -2293,6 +2328,7 @@ class AmazonConverseConfig(BaseConfig):
                     messages=messages,
                     model=capability_model,
                     deployment_model=model,
+                    model_info=model_info,
                     llm_provider="bedrock_converse",
                     user_continue_message=litellm_params.pop("user_continue_message", None),
                     base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)),
@@ -2345,8 +2381,11 @@ class AmazonConverseConfig(BaseConfig):
         litellm_params: dict[str, object],
         headers: dict | None = None,
     ) -> RequestObject:
+        from litellm.integrations.anthropic_cache_control_hook import routed_model_info
+
         base_model: Final = litellm_params.get("base_model")
         capability_model: Final = bedrock_capability_model(model, base_model)
+        model_info: Final = routed_model_info(litellm_params.get("model_info"))
         json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
         marked_messages: Final = self._messages_with_cache_marks(
             messages,
@@ -2355,6 +2394,7 @@ class AmazonConverseConfig(BaseConfig):
             drops_unsupported_params(litellm_params.get("drop_params")),
             json_optional_params,
             litellm_params,
+            model_info,
         )
         messages, system_content_blocks = self._transform_system_message(
             list(
@@ -2364,6 +2404,7 @@ class AmazonConverseConfig(BaseConfig):
             ),
             model=capability_model,
             deployment_model=model,
+            model_info=model_info,
         )
 
         # Convert last user message to guarded_text if guardrailConfig is present
@@ -2372,13 +2413,20 @@ class AmazonConverseConfig(BaseConfig):
             guard_input,
             optional_params,
             keeps_cache_marks=self._guarded_text_keeps_cache_marks(
-                guard_input, system_content_blocks, capability_model, model, json_optional_params, litellm_params
+                guard_input,
+                system_content_blocks,
+                capability_model,
+                model,
+                json_optional_params,
+                litellm_params,
+                model_info,
             ),
         )
 
         _data: Final[CommonRequestObject] = self._transform_request_helper(
             model=capability_model,
             deployment_model=model,
+            model_info=model_info,
             system_content_blocks=system_content_blocks,
             optional_params=optional_params,
             messages=messages,
@@ -2394,6 +2442,7 @@ class AmazonConverseConfig(BaseConfig):
                     messages=messages,
                     model=capability_model,
                     deployment_model=model,
+                    model_info=model_info,
                     llm_provider="bedrock_converse",
                     user_continue_message=litellm_params.pop("user_continue_message", None),
                     base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)),

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py

@@ -13,8 +13,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_anthropic_image_obj,
 )
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    RemoteMedia,
     async_inline_remote_media,
     convert_url_to_base64,
+    inline_every_remote_url,
 )
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.anthropic.common_utils import (
@@ -117,6 +119,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
     def should_strip_billing_metadata(self) -> bool:
         return True
 
+    def inlines_remote_media(self, media: RemoteMedia) -> bool:
+        return inline_every_remote_url(media)
+
     def get_supported_openai_params(self, model: str) -> list[str]:
         return AnthropicConfig.get_supported_openai_params(self, model)
 
@@ -129,10 +134,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         json_schema: Mapping[str, object] | None,
         request_params: Mapping[str, object],
     ) -> bool:
-        return bedrock_chat_uses_native_structured_output(
-            model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
+        return self.chat_uses_native_structured_output(
+            model, self.is_thinking_requested(request_params), self._deployment_model_info
         ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
 
+    def chat_uses_native_structured_output(
+        self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
+    ) -> bool:
+        return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info)
+
     def uses_opt_in_native_output(
         self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
     ) -> bool:
@@ -335,10 +345,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
 
         anthropic_request.pop("model", None)
         anthropic_request.pop("stream", None)
+        model_info: Final = deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params))
         apply_bedrock_invoke_structured_output(
             model=model,
             request_body=anthropic_request,
-            model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
+            model_info=model_info,
+            chat_uses_native_output=lambda: self.chat_uses_native_structured_output(
+                model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info
+            ),
         )
         strip_unsupported_bedrock_invoke_output_config_keys(
             model=model,

litellm/llms/bedrock/chat/mantle/transformation.py

@@ -15,7 +15,7 @@ from litellm.llms.anthropic.chat.transformation import JsonModeRequest
 from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import (
     AmazonAnthropicClaudeConfig,
 )
-from litellm.llms.bedrock.common_utils import build_mantle_messages_url
+from litellm.llms.bedrock.common_utils import bedrock_chat_opts_into_native_structured_output, build_mantle_messages_url
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ModelResponse
 
@@ -54,6 +54,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
             region=region,
         )
 
+    def chat_uses_native_structured_output(
+        self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
+    ) -> bool:
+        return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info)
+
     def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]:
         return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params))
 

litellm/llms/bedrock/common_utils.py

@@ -10,7 +10,7 @@ import json
 import os
 import re
 import warnings
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@@ -33,9 +33,11 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
 import litellm
 from litellm import verbose_logger
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
+from litellm.litellm_core_utils.core_helpers import lax_bool
 from litellm.litellm_core_utils.optional_imports import ensure_optional_import
 from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
 from litellm.litellm_core_utils.url_utils import ensure_api_route
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
@@ -315,7 +317,33 @@ def bedrock_supports_native_structured_output(
     ) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
 
 
-def bedrock_chat_uses_native_structured_output(
+def _bedrock_map_native_output_flag(model: str) -> object:
+    routed: Final = strip_bedrock_routing_prefix(model)
+    flags: Final = (
+        (entry or {}).get("supports_bedrock_native_structured_output")
+        for entry in (
+            _model_cost_entry(f"bedrock/{routed}"),
+            _model_cost_entry(routed),
+            _model_cost_entry(split_bedrock_region_path(model)[1]),
+        )
+    )
+    return next((flag for flag in flags if isinstance(flag, bool)), None)
+
+
+def _bedrock_native_output_opted_out(model: str, model_info: Mapping[str, object] | None) -> bool:
+    deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
+    if isinstance(deployment_flag, bool):
+        return not deployment_flag
+    return _bedrock_map_native_output_flag(model) is False
+
+
+def _bedrock_native_output_replaces_unforceable_tool(model: str, model_info: Mapping[str, object] | None) -> bool:
+    return AnthropicModelInfo.forced_tool_use_unsupported(model) and not _bedrock_native_output_opted_out(
+        model, model_info
+    )
+
+
+def bedrock_chat_opts_into_native_structured_output(
     model: str,
     thinking_requested: bool,
     custom_llm_provider: str = "bedrock",
@@ -326,7 +354,18 @@ def bedrock_chat_uses_native_structured_output(
     )
 
 
-def bedrock_native_output_is_opt_in(
+def bedrock_chat_uses_native_structured_output(
+    model: str,
+    thinking_requested: bool,
+    custom_llm_provider: str = "bedrock",
+    model_info: Mapping[str, object] | None = None,
+) -> bool:
+    return bedrock_chat_opts_into_native_structured_output(
+        model, thinking_requested, custom_llm_provider, model_info
+    ) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info))
+
+
+def bedrock_native_output_is_deployment_opt_in(
     model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
 ) -> bool:
     return not _bedrock_model_supports(
@@ -334,6 +373,15 @@ def bedrock_native_output_is_opt_in(
     ) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
 
 
+def bedrock_native_output_is_opt_in(
+    model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
+) -> bool:
+    return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
+        _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
+        or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
+    )
+
+
 def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
     return {
         **litellm_params,
@@ -424,13 +472,18 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
             return False
 
 
+def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
+    return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
+        anthropic_optional_params
+    )
+
+
 def requests_opt_in_native_output(
     model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None
 ) -> bool:
-    native_output_requested: Final = isinstance(
-        anthropic_optional_params.get("output_format"), Mapping
-    ) or requests_output_config_format(anthropic_optional_params)
-    return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info)
+    return requests_native_output(anthropic_optional_params) and bedrock_native_output_is_opt_in(
+        model, model_info=model_info
+    )
 
 
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object])
@@ -491,6 +544,7 @@ def apply_bedrock_invoke_structured_output(
     model: str,
     request_body: dict[str, object],  # mutable-ok: edited in place like siblings
     model_info: Mapping[str, object] | None = None,
+    chat_uses_native_output: Callable[[], bool] = lambda: False,
 ) -> None:
     """
     Route Anthropic structured-output params to what the Bedrock model supports.
@@ -502,7 +556,8 @@ def apply_bedrock_invoke_structured_output(
     forwarded as ``output_config.format``, which Bedrock relays to the model for
     enforced structured output. For every other model the schema is inlined into
     the last user message as best-effort text, with a warning because nothing
-    enforces it.
+    enforces it. ``chat_uses_native_output`` is asked only when a schema is
+    present, and a true answer sends it natively after the same cleanup.
     """
     legacy_output_format: Final = request_body.pop("output_format", None)
     output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body)
@@ -511,7 +566,15 @@ def apply_bedrock_invoke_structured_output(
     if schema_format is None:
         return
 
-    native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
+    native_format: Final = bedrock_native_output_format(
+        model,
+        schema_format,
+        model_info=(
+            {**(model_info or {}), "supports_bedrock_native_structured_output": True}
+            if chat_uses_native_output()
+            else model_info
+        ),
+    )
     if native_format is not None:
         existing_output_config: Final = request_body.get("output_config")
         if isinstance(existing_output_config, dict):
@@ -1453,7 +1516,9 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
     )
 
 
-def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str | None = None) -> bool:
+def bedrock_model_accepts_cache_points(
+    model: str | None, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool:
     """
     Whether Converse ``cachePoint`` blocks may be sent to this model.
 
@@ -1473,44 +1538,56 @@ def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str
     never silently degrade. ``litellm.utils.supports_prompt_caching`` is not reusable here:
     it returns False for unmapped models, the opposite polarity. ``deployment_model`` is the
     model as configured when ``model`` is its ``base_model``: its own declaration wins.
+    A routed call passes its deployment's ``model_info``, whose breakpoint flag wins outright and
+    whose ``supports_prompt_caching`` wins over the catalog's for a known model. Other flags come
+    from the catalog as loaded, since the registered keys are shared by every deployment of the
+    same model string and carry whichever sibling registered last.
     """
     if model is None:
         return True
     deployment_declared: Final = (
-        None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model)
+        None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model, model_info)
     )
     declared: Final = (
-        deployment_declared if deployment_declared is not None else bedrock_declared_cache_point_support(model)
+        deployment_declared
+        if deployment_declared is not None
+        else bedrock_declared_cache_point_support(model, model_info)
     )
     return declared is not False
 
 
-def bedrock_declared_cache_point_support(model: str) -> bool | None:
+def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None:
     if _OPENAI_FAMILY_MODEL_RE.search(model):
         return False
-    explicit_marker_support: Final = _explicit_cache_point_support(model)
+    explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
     if explicit_marker_support is not None:
         return explicit_marker_support
-    return _mapped_prompt_caching_support(model)
+    return _mapped_prompt_caching_support(model, model_info)
 
 
-def bedrock_model_known_to_accept_cache_points(model: str, deployment_model: str | None = None) -> bool:
+def bedrock_model_known_to_accept_cache_points(
+    model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool:
     """Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents
     as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI
     ``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for
     Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field."""
-    deployment_declared: Final = None if deployment_model is None else _declared_marker_support(deployment_model)
-    declared: Final = deployment_declared if deployment_declared is not None else _declared_marker_support(model)
+    deployment_declared: Final = (
+        None if deployment_model is None else _declared_marker_support(deployment_model, model_info)
+    )
+    declared: Final = (
+        deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info)
+    )
     return declared is True
 
 
-def _declared_marker_support(model: str) -> bool | None:
+def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
     if _OPENAI_FAMILY_MODEL_RE.search(model):
         return False
-    explicit_marker_support: Final = _explicit_cache_point_support(model)
+    explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
     if explicit_marker_support is not None:
         return explicit_marker_support
-    mapped: Final = _mapped_prompt_caching_support(model)
+    mapped: Final = _mapped_prompt_caching_support(model, model_info)
     if not _documents_cache_checkpoints(model):
         return None if mapped is None else False
     return mapped is not False
@@ -1520,30 +1597,43 @@ def _documents_cache_checkpoints(model: str) -> bool:
     return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova")
 
 
-def _explicit_cache_point_support(model: str) -> bool | None:
+def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
+    deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
+    if deployment_flag is not None:
+        return deployment_flag is True
+    catalog_entry: Final = _catalog_entry_reader(model_info)
     registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/"))
     return next(
         (
             entry.get("supports_prompt_cache_breakpoint") is True
             for key in (*registered_keys, model, get_bedrock_base_model(model))
-            if (entry := litellm.model_cost.get(key)) is not None
-            and entry.get("supports_prompt_cache_breakpoint") is not None
+            if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None
         ),
         None,
     )
 
 
-def _mapped_prompt_caching_support(model: str) -> bool | None:
+def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
+    catalog_entry: Final = _catalog_entry_reader(model_info)
     map_keys: Final = (model, get_bedrock_base_model(model))
-    if not any(key in litellm.model_cost for key in map_keys):
+    if not any(catalog_entry(key) is not None for key in map_keys):
         return None
+    deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching"))
+    if deployment_caching is not None:
+        return deployment_caching
     return any(
-        entry.get("supports_prompt_caching") is True
-        for key in map_keys
-        if (entry := litellm.model_cost.get(key)) is not None
+        entry.get("supports_prompt_caching") is True for key in map_keys if (entry := catalog_entry(key)) is not None
     )
 
 
+def _catalog_entry_reader(model_info: Mapping[str, object] | None) -> Callable[[str], Mapping[str, object] | None]:
+    from litellm.utils import catalog_with_runtime_registrations
+
+    if model_info is not None:
+        return catalog_with_runtime_registrations().get
+    return _model_cost_entry
+
+
 def bedrock_supports_tool_search(model: str) -> bool:
     """
     Whether Bedrock InvokeModel admits the ``tool_search_tool_*`` tool types on ``model``.

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py

@@ -40,6 +40,7 @@ from litellm.llms.bedrock.common_utils import (
     BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
     BedrockError,
     apply_bedrock_invoke_structured_output,
+    bedrock_native_output_is_deployment_opt_in,
     bedrock_supports_tool_search,
     ensure_bedrock_anthropic_messages_tool_names,
     get_anthropic_beta_from_headers,
@@ -47,7 +48,7 @@ from litellm.llms.bedrock.common_utils import (
     normalize_bedrock_opus_output_config_effort,
     normalize_custom_field_on_tools,
     normalize_tool_input_schema_types_for_bedrock_invoke,
-    requests_opt_in_native_output,
+    requests_native_output,
     requests_output_config_format,
     strip_unsupported_bedrock_invoke_output_config_keys,
     supports_bedrock_prompt_cache_ttl,
@@ -131,7 +132,10 @@ class AmazonAnthropicClaudeMessagesConfig(
         litellm_params: GenericLiteLLMParams,
     ) -> GenericLiteLLMParams | None:
         model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
-        if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info):
+        if not (
+            requests_native_output(anthropic_messages_optional_request_params)
+            and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
+        ):
             return None
         return litellm_params.model_copy(
             update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}

litellm/llms/bedrock/realtime/handler.py

@@ -9,7 +9,7 @@ import contextlib
 import importlib.metadata
 import json
 from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from types import MappingProxyType
 from typing import Final, NoReturn, Protocol, runtime_checkable
 
@@ -34,12 +34,15 @@ from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateR
 from litellm.types.llms.bedrock import AwsAuthParams
 from litellm.types.llms.openai import OpenAIRealtimeEvents
 from litellm.types.realtime import RealtimeResponseTransformInput
+from litellm.types.utils import StandardLoggingPayloadErrorInformation
 
 from ..base_aws_llm import BaseAWSLLM, run_aws_signing
 from ..common_utils import BedrockError
 from .transformation import BedrockRealtimeConfig, requested_modalities
 
 _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
+_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata")
 
 _EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
 
@@ -61,6 +64,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType(
 )
 
 
+def _stream_ended_before_session_closed() -> BedrockError:
+    return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
+
+
 def _as_bedrock_error(error: BaseException) -> BaseException:
     status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__)
     if status_code is None:
@@ -165,6 +172,12 @@ class _BridgeOutcome:
     client_disconnected: bool
 
 
+@dataclass(frozen=True, slots=True)
+class SessionEndSignals:
+    input_closing: asyncio.Event = field(default_factory=asyncio.Event)
+    output_failed: asyncio.Event = field(default_factory=asyncio.Event)
+
+
 async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]:
     if initial_message is not None:
         yield initial_message
@@ -188,8 +201,8 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
     raise BedrockError(
         status_code=400,
         message=(
-            "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
-            + (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
+            (f"{committed_failure}. " if committed_failure else "")
+            + "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
         ),
     )
 
@@ -201,11 +214,48 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
     return pending if isinstance(pending, str) else None
 
 
+def _spend_log_metadata(litellm_params: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None:
+    return next(
+        (
+            (key, metadata)
+            for key in _SPEND_LOG_METADATA_KEYS
+            if (metadata := _REQUEST_MAPPING_ADAPTER.validate_python(litellm_params.get(key)))
+        ),
+        None,
+    )
+
+
+def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None:
+    """The spend row of a session the provider ended reads failure while it still bills the forwarded events."""
+    litellm_params: Final = _REQUEST_MAPPING_ADAPTER.validate_python(
+        logging_obj.model_call_details.get("litellm_params")
+    )
+    if litellm_params is None:
+        return
+    spend_log_metadata: Final = _spend_log_metadata(litellm_params)
+    if spend_log_metadata is None:
+        return
+    status_code: Final[object] = getattr(error, "status_code", None)
+    error_information: Final[StandardLoggingPayloadErrorInformation] = {
+        "error_code": "" if status_code is None else str(status_code),
+        "error_class": type(error).__name__,
+        "error_message": redact_string(str(error)),
+        "llm_provider": "bedrock",
+    }
+    metadata_key, metadata = spend_log_metadata
+    logging_obj.model_call_details["litellm_params"] = {  # rebind-ok: the spend row reads the logging object
+        **litellm_params,
+        metadata_key: {**metadata, "status": "failure", "error_information": error_information},
+    }
+
+
 def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
     error: Final = _as_bedrock_error(failure)
     verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
     if _is_session_committed(scope):
         scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
+    if error is failure:
+        raise failure
     raise error from failure
 
 
@@ -267,6 +317,25 @@ async def _reject_session_update(
     return True
 
 
+async def _close_bedrock_input(
+    bedrock_stream: BedrockBidirectionalStream,
+    close_messages: Sequence[str],
+    send_to_bedrock: Callable[[str], Awaitable[None]],
+    timeout: float,
+) -> None:
+    async def flush_and_close() -> None:
+        for close_message in close_messages:
+            with contextlib.suppress(Exception):
+                await send_to_bedrock(close_message)
+        with contextlib.suppress(Exception):
+            await bedrock_stream.input_stream.close()
+
+    try:
+        await asyncio.wait_for(flush_and_close(), timeout)
+    except (asyncio.TimeoutError, TimeoutError):
+        verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout)
+
+
 class BedrockRealtime(BaseAWSLLM):
     """Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
 
@@ -278,6 +347,18 @@ class BedrockRealtime(BaseAWSLLM):
         super().__init__()
         self._sdk_version_lookup: Final = sdk_version_lookup
         self._close_timeout: Final = close_timeout
+        self._failed_input_closes: Final[set[asyncio.Task[None]]] = set()  # mutable-ok: strong refs to running closes
+
+    def _close_failed_input_in_background(
+        self,
+        bedrock_stream: BedrockBidirectionalStream,
+        send_to_bedrock: Callable[[str], Awaitable[None]],
+    ) -> None:
+        close_task: Final = asyncio.create_task(
+            _close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout)
+        )
+        self._failed_input_closes.add(close_task)
+        close_task.add_done_callback(self._failed_input_closes.discard)
 
     async def async_realtime(
         self,
@@ -433,23 +514,26 @@ class BedrockRealtime(BaseAWSLLM):
                 if _should_log_event(leftover_event)
             ),
         )
+        if outcome.provider_failure is not None and outcome.client_disconnected:
+            verbose_proxy_logger.debug(
+                "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
+            )
+        provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure
         if logged_events:
+            if provider_failure is not None:
+                _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure))
+            usage_logging_obj: Final = logging_obj if provider_failure is None else logging_obj.attempt_scoped_copy()
             GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
-                logging_obj.dispatch_success_handlers(
+                usage_logging_obj.dispatch_success_handlers(
                     list(logged_events),
                     prefer_async_handlers=True,
                 )
             )
             logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
 
-        if outcome.provider_failure is None:
+        if provider_failure is None:
             return
-        if outcome.client_disconnected:
-            verbose_proxy_logger.debug(
-                "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
-            )
-            return
-        _raise_provider_failure(websocket.scope, outcome.provider_failure)
+        _raise_provider_failure(websocket.scope, provider_failure)
 
     async def _bridge(
         self,
@@ -463,16 +547,34 @@ class BedrockRealtime(BaseAWSLLM):
     ) -> _BridgeOutcome:
         """Run both forwarding directions until the client leaves or either side fails."""
         logged: Final[list[OpenAIRealtimeEvents]] = []  # mutable-ok: events forwarded before a failure are still spend
+        signals: Final = SessionEndSignals()
 
         async def collect_logged_events() -> None:
-            async for event in self._forward_bedrock_to_client(
-                bedrock_stream, websocket, transformation_config, model, logging_obj, session_state
-            ):
-                logged.append(event)
+            try:
+                async for event in self._forward_bedrock_to_client(
+                    bedrock_stream,
+                    websocket,
+                    transformation_config,
+                    model,
+                    logging_obj,
+                    session_state,
+                    signals=signals,
+                ):
+                    logged.append(event)
+            except Exception:
+                signals.output_failed.set()
+                raise
 
         client_task: Final = asyncio.create_task(
             self._forward_client_to_bedrock(
-                websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message
+                websocket,
+                bedrock_stream,
+                transformation_config,
+                model,
+                session_state,
+                logging_obj,
+                initial_message,
+                signals=signals,
             )
         )
         bedrock_task: Final = asyncio.create_task(collect_logged_events())
@@ -506,6 +608,8 @@ class BedrockRealtime(BaseAWSLLM):
         session_state: RealtimeResponseTransformInput,
         logging_obj: LiteLLMLogging | None = None,
         initial_message: str | None = None,
+        *,
+        signals: SessionEndSignals,
     ) -> None:
         """Forward messages from client WebSocket to Bedrock stream.
 
@@ -521,7 +625,10 @@ class BedrockRealtime(BaseAWSLLM):
 
         async def send_to_bedrock(bedrock_message: str) -> None:
             event: Final = build_input_chunk(bedrock_message.encode("utf-8"))
-            await bedrock_stream.input_stream.send(event)
+            try:
+                await bedrock_stream.input_stream.send(event)
+            except Exception as e:
+                raise _stream_ended_before_session_closed() from e
             verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])
 
         try:
@@ -553,20 +660,13 @@ class BedrockRealtime(BaseAWSLLM):
                     client_ws, transformation_config, model, logging_obj, parsed_client_message
                 ):
                     break
+            signals.input_closing.set()
         finally:
-
-            async def close_bedrock_input() -> None:
-                for close_message in transformation_config.session_close_messages():
-                    with contextlib.suppress(Exception):
-                        await send_to_bedrock(close_message)
-                with contextlib.suppress(Exception):
-                    await bedrock_stream.input_stream.close()
-
-            try:
-                await asyncio.wait_for(close_bedrock_input(), self._close_timeout)
-            except (asyncio.TimeoutError, TimeoutError):
-                verbose_proxy_logger.debug(
-                    "Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout
+            if signals.output_failed.is_set():
+                self._close_failed_input_in_background(bedrock_stream, send_to_bedrock)
+            else:
+                await _close_bedrock_input(
+                    bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout
                 )
 
     async def _forward_bedrock_to_client(
@@ -577,6 +677,8 @@ class BedrockRealtime(BaseAWSLLM):
         model: str,
         logging_obj: LiteLLMLogging,
         session_state: RealtimeResponseTransformInput,
+        *,
+        signals: SessionEndSignals,
     ) -> AsyncIterator[OpenAIRealtimeEvents]:
         """Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
 
@@ -597,6 +699,8 @@ class BedrockRealtime(BaseAWSLLM):
             result = await output[1].receive()
 
             if result is None:
+                if not signals.input_closing.is_set():
+                    raise _stream_ended_before_session_closed()
                 verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
                 with contextlib.suppress(Exception):
                     await client_ws.close()

litellm/llms/cohere/embed/handler.py

@@ -82,7 +82,7 @@ async def async_embedding(
         )
 
     try:
-        response: Final = await client.post(api_base, headers=headers, data=json.dumps(data))
+        response: Final = await client.post(api_base, headers=headers, data=json.dumps(data), timeout=timeout)
     except httpx.HTTPStatusError as e:
         ## LOGGING
         logging_obj.post_call(
@@ -166,7 +166,7 @@ def embedding(
     if client is None or not isinstance(client, HTTPHandler):
         client = HTTPHandler(concurrent_limit=1)
 
-    response: Final = client.post(embed_url, headers=headers, data=json.dumps(data))
+    response: Final = client.post(embed_url, headers=headers, data=json.dumps(data), timeout=timeout)
 
     return CohereEmbeddingConfig().transform_response(
         response=response,

litellm/llms/custom_httpx/llm_http_handler.py

@@ -37,8 +37,13 @@ import litellm
 import litellm.litellm_core_utils
 import litellm.types
 import litellm.types.utils
+from litellm._internal_context import internal_sub_call
 from litellm._logging import redact_string, verbose_logger
-from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, update_headers_with_filtered_beta
+from litellm.anthropic_beta_headers_manager import (
+    aload_beta_headers_config,
+    resolve_beta_provider,
+    update_headers_with_filtered_beta,
+)
 from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.files.types import FileContentStreamingResult
 from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@@ -455,6 +460,20 @@ def _google_genai_streaming_hidden_params(
     }
 
 
+def _log_generate_content_request(
+    logging_obj: LiteLLMLoggingObj,
+    contents: object,
+    api_base: str,
+    headers: Mapping[str, object],
+    body: Mapping[str, object],
+) -> None:
+    logging_obj.pre_call(
+        input=contents,
+        api_key="",
+        additional_args={"complete_input_dict": body, "api_base": api_base, "headers": headers},
+    )
+
+
 @lru_cache(maxsize=None)
 def _responses_api_optional_request_param_names() -> frozenset[str]:
     return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
@@ -2581,14 +2600,16 @@ class BaseLLMHTTPHandler:
             litellm_params.get("additional_drop_params") or (),
             anthropic_messages_optional_request_params,
         )
-        if (
-            request_params.get("safeguards") is not None
-            and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
-        ):
-            await aload_beta_headers_config()
         affinity_api_base: Final = (
             AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
         )
+        takes_another_beta_row: Final = (
+            resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
+        )
+        provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
+        filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row
+        if request_params.get("safeguards") is not None and filters_beta_headers:
+            await aload_beta_headers_config()
         default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
         (
             headers,
@@ -2607,8 +2628,14 @@ class BaseLLMHTTPHandler:
             api_base=api_base,
         )
 
-        if anthropic_messages_provider_config.should_filter_anthropic_beta_headers():
-            headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider)
+        if filters_beta_headers and headers.get("anthropic-beta") is not None:
+            await aload_beta_headers_config()
+            headers = update_headers_with_filtered_beta(
+                headers=headers,
+                provider=custom_llm_provider,
+                api_base=affinity_api_base,
+                unlisted="drop" if provider_filters_betas else "forward",
+            )
 
         from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 
@@ -6226,12 +6253,13 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses(
-                model=patch.model or model,
-                input=patch.messages,  # pyright: ignore[reportArgumentType]  # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime
-                **optional_params,
-                **kwargs_for_followup,
-            )
+            with internal_sub_call(logging_obj.litellm_call_id):
+                response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses(
+                    model=patch.model or model,
+                    input=patch.messages,  # pyright: ignore[reportArgumentType]  # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime
+                    **optional_params,
+                    **kwargs_for_followup,
+                )
 
             if callback is not None:
                 try:
@@ -6320,6 +6348,7 @@ class BaseLLMHTTPHandler:
         max_loops: int,
         fingerprints: list[str],
         fingerprint: str,
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
     ) -> ModelResponse | CustomStreamWrapper:
         patch: Final = plan.request_patch or AgenticLoopRequestPatch()
         if patch.messages is None:
@@ -6359,12 +6388,14 @@ class BaseLLMHTTPHandler:
             fingerprint=fingerprint,
         )
 
-        return await litellm.acompletion(
-            model=full_model_name,
-            messages=patch.messages,
-            **optional_params_for_followup,
-            **kwargs_for_followup,
-        )
+        with internal_sub_call(logging_obj.litellm_call_id if logging_obj is not None else None):
+            response: Final = await litellm.acompletion(
+                model=full_model_name,
+                messages=patch.messages,
+                **optional_params_for_followup,
+                **kwargs_for_followup,
+            )
+        return response
 
     def _maybe_wrap_in_fake_stream(
         self,
@@ -6713,6 +6744,7 @@ class BaseLLMHTTPHandler:
                     messages=messages,
                     optional_params=optional_params,
                     kwargs=kwargs_with_provider,
+                    logging_obj=logging_obj,
                     custom_llm_provider=custom_llm_provider,
                     depth=depth,
                     max_loops=max_loops,
@@ -7065,7 +7097,7 @@ class BaseLLMHTTPHandler:
         timeout: float | httpx.Timeout,
         provider_config: BaseRealtimeHTTPConfig | None = None,
         model: str | None = None,
-        extra_headers: dict[str, object] | None = None,
+        extra_headers: Mapping[str, object] | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         api_version: str | None = None,
     ) -> httpx.Response:
@@ -7098,7 +7130,7 @@ class BaseLLMHTTPHandler:
         timeout: float | httpx.Timeout,
         provider_config: BaseRealtimeHTTPConfig | None = None,
         model: str | None = None,
-        extra_headers: dict[str, object] | None = None,
+        extra_headers: Mapping[str, object] | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         api_version: str | None = None,
     ) -> httpx.Response:
@@ -7117,9 +7149,8 @@ class BaseLLMHTTPHandler:
             api_version=api_version,
         )
 
-    async def _async_realtime_session_post(
+    async def async_realtime_translation_client_secret_handler(
         self,
-        endpoint: Literal["client_secrets", "transcription_sessions"],
         api_base: str,
         api_key: str,
         request_data: dict[str, object],
@@ -7127,7 +7158,35 @@ class BaseLLMHTTPHandler:
         timeout: float | httpx.Timeout,
         provider_config: BaseRealtimeHTTPConfig | None = None,
         model: str | None = None,
-        extra_headers: dict[str, object] | None = None,
+        extra_headers: Mapping[str, object] | None = None,
+        client: HTTPHandler | AsyncHTTPHandler | None = None,
+        api_version: str | None = None,
+    ) -> httpx.Response:
+        return await self._async_realtime_session_post(
+            endpoint="translation_client_secrets",
+            api_base=api_base,
+            api_key=api_key,
+            request_data=request_data,
+            logging_obj=logging_obj,
+            timeout=timeout,
+            provider_config=provider_config,
+            model=model,
+            extra_headers=extra_headers,
+            client=client,
+            api_version=api_version,
+        )
+
+    async def _async_realtime_session_post(
+        self,
+        endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"],
+        api_base: str,
+        api_key: str,
+        request_data: dict[str, object],
+        logging_obj: LiteLLMLoggingObj,
+        timeout: float | httpx.Timeout,
+        provider_config: BaseRealtimeHTTPConfig | None = None,
+        model: str | None = None,
+        extra_headers: Mapping[str, object] | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         api_version: str | None = None,
     ) -> httpx.Response:
@@ -7150,13 +7209,20 @@ class BaseLLMHTTPHandler:
                 url = provider_config.get_transcription_session_url(
                     api_base=api_base, model=model or "", api_version=api_version
                 )
+            elif endpoint == "translation_client_secrets":
+                url = provider_config.get_translation_client_secret_url(
+                    api_base=api_base, model=model or "", api_version=api_version
+                )
             else:
                 url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version)
             headers: dict[str, object] = provider_config.validate_environment(
                 headers={}, model=model or "", api_key=api_key
             )
         else:
-            url = join_api_base(api_base, f"v1/realtime/{endpoint}")
+            endpoint_path: Final = (
+                "translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint
+            )
+            url = join_api_base(api_base, f"v1/realtime/{endpoint_path}")
             headers = {
                 "Authorization": f"Bearer {api_key}",
                 "Content-Type": "application/json",
@@ -7190,6 +7256,12 @@ class BaseLLMHTTPHandler:
                 )
             raise
 
+    @staticmethod
+    def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler:
+        if isinstance(client, AsyncHTTPHandler):
+            return client
+        return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI)
+
     async def async_realtime_calls_handler(
         self,
         api_base: str,
@@ -7199,10 +7271,11 @@ class BaseLLMHTTPHandler:
         timeout: float | httpx.Timeout,
         provider_config: BaseRealtimeHTTPConfig | None = None,
         model: str | None = None,
-        session_config: dict[str, object] | None = None,
-        extra_headers: dict[str, object] | None = None,
+        session_config: Mapping[str, object] | None = None,
+        extra_headers: Mapping[str, object] | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         api_version: str | None = None,
+        translation: bool = False,
     ) -> httpx.Response:
         """
         Forward POST /v1/realtime/calls (SDP exchange) to upstream provider.
@@ -7214,18 +7287,30 @@ class BaseLLMHTTPHandler:
           - sdp: the SDP offer (text)
           - session: JSON string with {"type": "realtime", "model": "...", ...}
         """
-        if client is None or not isinstance(client, AsyncHTTPHandler):
-            async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders.OPENAI,
-            )
-        else:
-            async_httpx_client = client
+        session_data: Final[dict[str, object]] = {  # mutable-ok: model and session type are resolved locally
+            **(session_config or {})
+        }
+        if "type" not in session_data:
+            session_data["type"] = "translation" if translation else "realtime"
+        if "model" not in session_data and model:
+            session_data["model"] = model
+
+        sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body
+
+        async_httpx_client: Final = self._get_realtime_async_http_client(client)
 
         if provider_config is not None:
-            url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version)
+            url = (
+                provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version)
+                if translation
+                else provider_config.get_realtime_calls_url(
+                    api_base=api_base, model=model or "", api_version=api_version
+                )
+            )
             headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key)
         else:
-            url = join_api_base(api_base, "v1/realtime/calls")
+            path: Final = "translations/calls" if translation else "calls"
+            url = join_api_base(api_base, f"v1/realtime/{path}")
             headers = {
                 "Authorization": f"Bearer {openai_ephemeral_key}",
             }
@@ -7233,14 +7318,8 @@ class BaseLLMHTTPHandler:
         if extra_headers:
             headers.update(extra_headers)
 
-        # Build multipart form data: sdp + session JSON
-        session_data: Final = session_config or {}
-        if "type" not in session_data:
-            session_data["type"] = "realtime"
-        if "model" not in session_data and model:
-            session_data["model"] = model
-
-        sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body
+        if translation:
+            headers["Content-Type"] = "application/sdp"
 
         files: Final = {
             "sdp": (None, sdp_text, "text/plain"),
@@ -7258,12 +7337,14 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            return await async_httpx_client.post(
-                url=url,
-                headers=headers,
-                files=files,
-                timeout=timeout,
-            )
+            if translation:
+                return await async_httpx_client.post(
+                    url=url,
+                    headers=headers,
+                    content=sdp_text,
+                    timeout=timeout,
+                )
+            return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout)
         except Exception as e:
             if provider_config is not None:
                 raise self._handle_error(
@@ -8073,7 +8154,7 @@ class BaseLLMHTTPHandler:
         """
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -8269,7 +8350,7 @@ class BaseLLMHTTPHandler:
         """
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -8450,7 +8531,7 @@ class BaseLLMHTTPHandler:
         """
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -8614,7 +8695,7 @@ class BaseLLMHTTPHandler:
     ):
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -8755,7 +8836,7 @@ class BaseLLMHTTPHandler:
     ):
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -8928,7 +9009,7 @@ class BaseLLMHTTPHandler:
     ):
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -9115,7 +9196,7 @@ class BaseLLMHTTPHandler:
     ):
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -9248,7 +9329,7 @@ class BaseLLMHTTPHandler:
         """
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -9330,7 +9411,7 @@ class BaseLLMHTTPHandler:
         """
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -9515,7 +9596,7 @@ class BaseLLMHTTPHandler:
         """
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
+                llm_provider=custom_llm_provider,
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
@@ -12443,6 +12524,10 @@ class BaseLLMHTTPHandler:
         from litellm.google_genai.streaming_iterator import (
             GoogleGenAIGenerateContentStreamingIterator,
         )
+        from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
+            post_with_accepted_thought_signatures_sync,
+            with_placeholder_native_thought_signatures,
+        )
 
         if _is_async:
             return self.async_generate_content_handler(
@@ -12494,27 +12579,19 @@ class BaseLLMHTTPHandler:
         if extra_body:
             data.update(extra_body)
 
-        ## LOGGING
-        logging_obj.pre_call(
-            input=contents,
-            api_key="",
-            additional_args={
-                "complete_input_dict": data,
-                "api_base": api_base,
-                "headers": headers,
-            },
-        )
+        log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
+        log_request(data)
+
+        def post(body: Mapping[str, object]) -> httpx.Response:
+            return sync_httpx_client.post(
+                url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
+            )
 
         try:
+            sent_body, response = post_with_accepted_thought_signatures_sync(
+                data, with_placeholder_native_thought_signatures, None, post, log_request
+            )
             if stream:
-                response = sync_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                    stream=True,
-                )
-                # Return streaming iterator
                 return GoogleGenAIGenerateContentStreamingIterator(
                     response=response,
                     model=model,
@@ -12522,7 +12599,7 @@ class BaseLLMHTTPHandler:
                     generate_content_provider_config=generate_content_provider_config,
                     litellm_metadata=litellm_metadata or {},
                     custom_llm_provider=custom_llm_provider,
-                    request_body=data,
+                    request_body=dict(sent_body),
                     hidden_params=_google_genai_streaming_hidden_params(
                         api_base=api_base,
                         litellm_params=litellm_params,
@@ -12530,13 +12607,6 @@ class BaseLLMHTTPHandler:
                         response_headers=response.headers,
                     ),
                 )
-            else:
-                response = sync_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                )
         except Exception as e:
             raise self._handle_error(
                 e=e,
@@ -12574,6 +12644,10 @@ class BaseLLMHTTPHandler:
         from litellm.google_genai.streaming_iterator import (
             AsyncGoogleGenAIGenerateContentStreamingIterator,
         )
+        from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
+            post_with_accepted_thought_signatures_async,
+            with_placeholder_native_thought_signatures,
+        )
 
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
@@ -12609,27 +12683,19 @@ class BaseLLMHTTPHandler:
         if extra_body:
             data.update(extra_body)
 
-        ## LOGGING
-        logging_obj.pre_call(
-            input=contents,
-            api_key="",
-            additional_args={
-                "complete_input_dict": data,
-                "api_base": api_base,
-                "headers": headers,
-            },
-        )
+        log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
+        log_request(data)
+
+        async def post(body: Mapping[str, object]) -> httpx.Response:
+            return await async_httpx_client.post(
+                url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
+            )
 
         try:
+            sent_body, response = await post_with_accepted_thought_signatures_async(
+                data, with_placeholder_native_thought_signatures, None, post, log_request
+            )
             if stream:
-                response = await async_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                    stream=True,
-                )
-                # Return async streaming iterator
                 return AsyncGoogleGenAIGenerateContentStreamingIterator(
                     response=response,
                     model=model,
@@ -12637,7 +12703,7 @@ class BaseLLMHTTPHandler:
                     generate_content_provider_config=generate_content_provider_config,
                     litellm_metadata=litellm_metadata or {},
                     custom_llm_provider=custom_llm_provider,
-                    request_body=data,
+                    request_body=dict(sent_body),
                     hidden_params=_google_genai_streaming_hidden_params(
                         api_base=api_base,
                         litellm_params=litellm_params,
@@ -12645,13 +12711,6 @@ class BaseLLMHTTPHandler:
                         response_headers=response.headers,
                     ),
                 )
-            else:
-                response = await async_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                )
         except Exception as e:
             raise self._handle_error(
                 e=e,

litellm/llms/huggingface/embedding/handler.py

@@ -38,7 +38,9 @@ class _SupportsTokenEncode(Protocol):
     def encode(self, text: str) -> list[int]: ...
 
 
-def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object:
+def get_hf_task_embedding_for_model(
+    model: str, task_type: str | None, api_base: str, timeout: float | httpx.Timeout | None = None
+) -> object:
     if task_type is not None:
         if task_type in get_args(hf_tasks_embeddings):
             return task_type
@@ -46,7 +48,7 @@ def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base:
             raise Exception(f"Invalid task_type={task_type}. Expected one of={hf_tasks_embeddings}")
     http_client: Final = HTTPHandler(concurrent_limit=1)
 
-    model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}")
+    model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}", timeout=timeout)
 
     model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json()))
 
@@ -148,6 +150,7 @@ class HuggingFaceEmbedding(BaseLLM):
         call_type: Literal["sync", "async"],
         optional_params: dict,
         embed_url: str,
+        timeout: float | httpx.Timeout | None = None,
     ) -> dict:
         data: dict = {}
 
@@ -165,7 +168,9 @@ class HuggingFaceEmbedding(BaseLLM):
             task_type: Final = optional_params.pop("input_type", None)
 
             if call_type == "sync":
-                hf_task: Final = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL)
+                hf_task: Final = get_hf_task_embedding_for_model(
+                    model=model, task_type=task_type, api_base=HF_HUB_URL, timeout=timeout
+                )
             elif call_type == "async":
                 return self._async_transform_input(model=model, task_type=task_type, embed_url=embed_url, input=input)
 
@@ -253,6 +258,7 @@ class HuggingFaceEmbedding(BaseLLM):
             call_type="sync",
             optional_params=optional_params,
             embed_url=api_base,
+            timeout=timeout,
         )
 
         ## LOGGING
@@ -269,9 +275,10 @@ class HuggingFaceEmbedding(BaseLLM):
         if client is None:
             client = get_async_httpx_client(
                 llm_provider=litellm.LlmProviders.HUGGINGFACE,
+                params={"timeout": timeout},
             )
 
-        response: Final = await client.post(api_base, headers=headers, data=json.dumps(data))
+        response: Final = await client.post(api_base, headers=headers, data=json.dumps(data), timeout=timeout)
 
         ## LOGGING
         logging_obj.post_call(
@@ -321,7 +328,9 @@ class HuggingFaceEmbedding(BaseLLM):
             litellm_params=litellm_params,
         )
         task_type: Final = optional_params.get("input_type", None)
-        task: Final = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL)
+        task: Final = get_hf_task_embedding_for_model(
+            model=model, task_type=task_type, api_base=HF_HUB_URL, timeout=timeout
+        )
         # print_verbose(f"{model}, {task}")
         embed_url = ""
         if model.startswith(("http://", "https://")):
@@ -359,6 +368,7 @@ class HuggingFaceEmbedding(BaseLLM):
             call_type="sync",
             optional_params=optional_params,
             embed_url=embed_url,
+            timeout=timeout,
         )
 
         ## LOGGING
@@ -374,7 +384,7 @@ class HuggingFaceEmbedding(BaseLLM):
         ## COMPLETION CALL
         if client is None or not isinstance(client, HTTPHandler):
             client = HTTPHandler(concurrent_limit=1)
-        response: Final = client.post(embed_url, headers=headers, data=json.dumps(data))
+        response: Final = client.post(embed_url, headers=headers, data=json.dumps(data), timeout=timeout)
 
         ## LOGGING
         logging_obj.post_call(

litellm/llms/openai/chat/gpt_transformation.py

@@ -556,18 +556,23 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
     ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
         """
         A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
-        api.openai.com, or one opted in through ``prompt_cache_options`` or the cost
-        map) gets each Anthropic ``cache_control`` mark rewritten in place, and under
-        drop_params each content part's ``prompt_cache_breakpoint`` cut down to what chat
-        completions accepts. Any other endpoint keeps or strips the marks as before, and
-        under drop_params also loses every ``prompt_cache_breakpoint`` it cannot read.
+        api.openai.com, or one opted in through ``prompt_cache_options``, its own
+        ``model_info``, or the cost map) gets each
+        Anthropic ``cache_control`` mark rewritten in place, and under drop_params each
+        content part's ``prompt_cache_breakpoint`` cut down to what chat completions
+        accepts. Any other endpoint keeps or strips the marks as before, and under
+        drop_params also loses every ``prompt_cache_breakpoint`` it cannot read.
         """
-        from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+        from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, routed_model_info
 
         custom_llm_provider: Final = _litellm_params_str(litellm_params, "custom_llm_provider")
         api_base: Final = _litellm_params_str(litellm_params, "api_base")
         reads_breakpoints: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-            model, custom_llm_provider, api_base, prompt_cache_options
+            model,
+            custom_llm_provider,
+            api_base,
+            prompt_cache_options,
+            routed_model_info(litellm_params.get("model_info")),
         )
         if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
             return (

litellm/llms/openai/decisions/transformation.py

@@ -307,10 +307,13 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) ->
 
 
 def ir_to_openai_response(
-    response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str
+    response: DecisionsIRResponse,
+    request: DecisionsIRRequest,
+    requested_model: str,
+    evidence_fields: frozenset[str] = frozenset(),
 ) -> OpenAIDecisionResponse:
     usage: Final = response.usage or DecisionsIRUsage()
-    return OpenAIDecisionResponse(
+    formatted: Final = OpenAIDecisionResponse(
         model=response.model or requested_model,
         answers=tuple(
             _openai_answer(question, answer)
@@ -327,6 +330,7 @@ def ir_to_openai_response(
             total_tokens=usage.input_tokens + usage.output_tokens,
         ),
     )
+    return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()})
 
 
 class OpenAIDecisionsConfig(BaseDecisionsConfig):

litellm/llms/openai/live/handler.py

@@ -18,6 +18,7 @@ from litellm._logging import redact_secrets
 from litellm.constants import (
     OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS,
     OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
+    REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
     REALTIME_SESSION_SUCCESS_LOGGED_KEY,
     REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
 )
@@ -316,12 +317,15 @@ async def _forward_live_backend_frames(
     backend_websocket: LiveBackendWebSocket,
     client_gone: asyncio.Event,
     event_retention: _LiveEventRetention,
+    upstream_ended_abnormally: asyncio.Event,
 ) -> None:
     from starlette.websockets import WebSocketDisconnect
     from websockets.exceptions import ConnectionClosed
 
     event_loop: Final = asyncio.get_running_loop()
-    async for message, close_deadline in _live_backend_messages(websocket, backend_websocket):
+    async for message, close_deadline in _live_backend_messages(
+        websocket, backend_websocket, upstream_ended_abnormally
+    ):
         if (event := _metering_event(message)) is not None:
             event_retention.retain(event)
         if client_gone.is_set():
@@ -341,6 +345,7 @@ async def _forward_live_backend_frames(
 async def _live_backend_messages(
     websocket: LiveClientWebSocket,
     backend_websocket: LiveBackendWebSocket,
+    upstream_ended_abnormally: asyncio.Event,
 ) -> AsyncIterator[tuple[str, float | None]]:
     from starlette.websockets import WebSocketDisconnect
     from websockets.exceptions import ConnectionClosed
@@ -350,6 +355,8 @@ async def _live_backend_messages(
             yield message, None
     except ConnectionClosed as error:
         upstream_close: Final = backend_close_from(error)
+        if not upstream_close.ended_cleanly:
+            upstream_ended_abnormally.set()
         with suppress(WebSocketDisconnect, RuntimeError, ConnectionClosed, OSError):
             await websocket.close(code=upstream_close.code, reason=upstream_close.reason)
         return
@@ -434,6 +441,7 @@ class OpenAILiveSessions(OpenAIRealtime):
             "complete_input_dict": {"session_start": rewritten_start},
         }
         event_retention: Final = _LiveEventRetention()
+        session_ended_abnormally: Final = asyncio.Event()
         try:
             logging_obj.pre_call(input=None, api_key=api_key, additional_args=pre_call_args)
             async with self._websocket_connector(
@@ -448,7 +456,11 @@ class OpenAILiveSessions(OpenAIRealtime):
                         websocket=websocket,
                         backend_websocket=backend_websocket,
                         event_retention=event_retention,
+                        upstream_ended_abnormally=session_ended_abnormally,
                     )
+                except Exception:
+                    session_ended_abnormally.set()
+                    raise
                 finally:
                     session_end_time: Final = self._clock()
                     retained_events: Final = event_retention.results()
@@ -463,6 +475,10 @@ class OpenAILiveSessions(OpenAIRealtime):
                     logging_obj.model_call_details[  # rebind-ok: flags the shared logging object as already logged
                         REALTIME_SESSION_SUCCESS_LOGGED_KEY
                     ] = True
+                    if session_ended_abnormally.is_set():
+                        logging_obj.model_call_details[  # rebind-ok: flags the shared logging object for the router
+                            REALTIME_SESSION_ENDED_ABNORMALLY_KEY
+                        ] = True
         except InvalidStatus as error:
             await close_after_upstream_handshake_refusal(websocket, error.response.status_code)
         except Exception as error:  # noqa: BLE001  # any relay failure must still close the client socket with a redacted reason
@@ -484,6 +500,7 @@ class OpenAILiveSessions(OpenAIRealtime):
         websocket: LiveClientWebSocket,
         backend_websocket: LiveBackendWebSocket,
         event_retention: _LiveEventRetention,
+        upstream_ended_abnormally: asyncio.Event,
     ) -> None:
         from websockets.exceptions import ConnectionClosed
 
@@ -503,6 +520,7 @@ class OpenAILiveSessions(OpenAIRealtime):
                 backend_websocket=backend_websocket,
                 client_gone=client_gone,
                 event_retention=event_retention,
+                upstream_ended_abnormally=upstream_ended_abnormally,
             )
         )
         client_gone_task: Final = asyncio.create_task(client_gone.wait())