Merge remote-tracking branch 'origin/main' into HEAD

# Conflicts:
#	tests/integration/pricing/test_price_data_reload_prices.py

.circleci/scripts/run_integration.sh

@@ -128,6 +128,8 @@ fi
 sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
 awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
 
+export UV_OFFLINE=true
+
 if [ "$mode" = parity ]; then
   export INTEGRATION_WORKERS=1
 else

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -4,9 +4,21 @@
      Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
      "## Affected release" with nothing under it must not appear in the final description -->
 
-<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
+## Handwritten 1-3 sentence description
+
+<!-- The author writes a plain summary in their own words before requesting review.
+     Wrap newly AI-written summary text in the make-me-human-readable tags below, including backticks.
+     The author reviews or rewrites the draft and removes the tags.
+     When updating an existing PR, preserve the author's wording and removed tags.
+     Changes elsewhere in the body do not make an unchanged summary an AI draft again.
+     Add tags again only around newly AI-written replacement summary text, never merely to match this template.
+     This preserves author review across later updates (2026-11-02, #49779).
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
-     No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
+     No bullets, tables, or code names here. Everything else goes inside the Details block below -->
+
+`<make-me-human-readable>`
+<blah>
+`</make-me-human-readable>`
 
 <details>
 <summary>Details</summary>

.github/workflows/compat-matrix-image.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - tests/e2e/claude_code/cron_vm/**
-      - tests/e2e/claude_code/pr_gate_version_resolver.py
+      - tests/e2e/cli_harness/npm_version_resolver.py
       - .github/workflows/compat-matrix-image.yml
   workflow_dispatch:
 
@@ -36,7 +36,7 @@ jobs:
             whoami
             gh --version
             uv --version
-            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
+            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
             /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
             /tmp/claude-cli/claude --version
           '

.github/workflows/create-rc-branch.yml

@@ -2,7 +2,7 @@ name: Create RC Branch
 
 on:
   schedule:
-    - cron: "0 3 * * 5"
+    - cron: "0 3 * * 1"
       timezone: "America/Los_Angeles"
   workflow_dispatch:
 

.github/workflows/deploy-dev-on-main.yml

@@ -1,30 +0,0 @@
-name: Deploy main to gateway-dev
-
-on:
-  push:
-    branches: [main]
-
-permissions: {}
-
-concurrency:
-  group: deploy-dev-dispatch
-  cancel-in-progress: true
-
-jobs:
-  dispatch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
-        id: token
-        with:
-          app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
-          private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
-          owner: BerriAI
-          repositories: litellm-ops
-          permission-actions: write
-      - name: Dispatch the litellm-ops dev deploy
-        env:
-          GH_TOKEN: ${{ steps.token.outputs.token }}
-        run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

.github/workflows/test-linting.yml

@@ -208,7 +208,7 @@ jobs:
             exit 0
           fi
           retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
+          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
           tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
           PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
           uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \

.github/workflows/test-unit.yml

@@ -441,6 +441,7 @@ jobs:
               tests/unit/ocr
               tests/unit/passthrough
               tests/unit/rag
+              tests/unit/cookbook
               tests/unit/realtime_api
               tests/unit/rerank_api
               tests/unit/search

.github/workflows/weekly_load_anomaly.yml

@@ -1,85 +0,0 @@
-name: "Weekly Load Anomaly Check"
-
-on:
-  schedule:
-    - cron: "0 12 * * 6"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  weekly-load-anomaly:
-    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    services:
-      postgres:
-        image: postgres:16.6
-        env:
-          POSTGRES_USER: llmproxy
-          POSTGRES_PASSWORD: dbpassword9090
-          POSTGRES_DB: litellm
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U llmproxy"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
-    env:
-      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
-      LITELLM_MASTER_KEY: sk-weekly-anomaly-check
-      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache the Rust build
-        uses: ./.github/actions/cache-cargo-build
-
-      - name: Install dependencies
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
-
-      - name: Cache Prisma binaries
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Start the proxy
-        run: |
-          nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
-          for _ in $(seq 1 90); do
-            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "proxy never became live"
-          tail -n 100 proxy.log
-          exit 1
-
-      - name: Run the weekly session anomaly test
-        env:
-          E2E_WEEKLY_ANOMALY: "1"
-        run: |
-          uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
-
-      - name: Show proxy log on failure
-        if: failure()
-        run: tail -n 300 proxy.log

AGENTS.md

@@ -41,6 +41,8 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
+Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`, and swap its `--config` for a copy of `litellm/proxy/dev_config.yaml` that points both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. Check that two worker PIDs are serving, and use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload`. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker check for it. Say in the proof which part you dropped and why
+
 Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
 
 For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce

README.md

@@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) |  |  |  | ✅ |  |  |  |  |  |  |
 | [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) |  |  |  |  |  |  |  |  |  |  |
 | [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -696,7 +697,7 @@ For companies that need better security, user management and professional suppor
 [Talk to founders](https://enterprise.litellm.ai/demo)
 
 This covers:
-- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
+- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
 - ✅ **Professional Support - Dedicated discord + slack**

cookbook/gpt_realtime_translate.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+import sys
+import wave
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlencode, urlsplit, urlunsplit
+
+import websockets
+from websockets.asyncio.client import ClientConnection
+
+SAMPLE_RATE = 24_000
+CHANNELS = 1
+SAMPLE_WIDTH = 2
+CHUNK_DURATION_SECONDS = 0.1
+CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
+RESPONSE_TIMEOUT_SECONDS = 30.0
+AUDIO_EVENT_TYPES = frozenset(
+    {
+        "session.output_audio.delta",
+        "response.audio.delta",
+        "response.output_audio.delta",
+    }
+)
+TRANSCRIPT_EVENT_TYPES = frozenset(
+    {
+        "session.output_transcript.delta",
+        "response.text.delta",
+        "response.output_audio_transcript.delta",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    input_wav: Path
+    output_wav: Path
+    base_url: str
+    model: str
+    target_language: str
+    trailing_silence_seconds: float
+    api_key: str
+
+
+def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
+    sys.stdout.write(f"{message}{end}")
+    if flush:
+        sys.stdout.flush()
+
+
+def write_stderr(message: str) -> None:
+    sys.stderr.write(f"{message}\n")
+
+
+def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
+    parser = argparse.ArgumentParser(
+        description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
+    )
+    parser.add_argument("input_wav", type=Path)
+    parser.add_argument("--output", type=Path, default=Path("translated.wav"))
+    parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
+    parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
+    parser.add_argument("--target-language", default="fr")
+    parser.add_argument("--trailing-silence", type=float, default=1.5)
+    parsed = parser.parse_args(argv)
+    api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
+    if parsed.trailing_silence < 0:
+        return "--trailing-silence must be zero or greater"
+    return Settings(
+        input_wav=parsed.input_wav,
+        output_wav=parsed.output,
+        base_url=parsed.base_url,
+        model=parsed.model,
+        target_language=parsed.target_language,
+        trailing_silence_seconds=parsed.trailing_silence,
+        api_key=api_key,
+    )
+
+
+def translation_url(base_url: str, model: str) -> str | None:
+    parsed = urlsplit(base_url.rstrip("/"))
+    scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
+    if not scheme or not parsed.netloc:
+        return None
+    base_path = parsed.path.rstrip("/")
+    realtime_path = (
+        f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
+    )
+    return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
+
+
+def read_pcm16_wav(path: Path) -> bytes | str:
+    try:
+        with wave.open(str(path), "rb") as source:
+            actual_format = (
+                source.getnchannels(),
+                source.getsampwidth(),
+                source.getframerate(),
+                source.getcomptype(),
+            )
+            expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
+            if actual_format != expected_format:
+                return (
+                    f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
+                    f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
+                    f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
+                )
+            return source.readframes(source.getnframes())
+    except (OSError, EOFError, wave.Error) as exc:
+        return f"Unable to read {path}: {exc}"
+
+
+def audio_chunks(audio: bytes) -> Iterator[bytes]:
+    return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
+
+
+def audio_message(audio: bytes) -> str:
+    return json.dumps(
+        {
+            "type": "session.input_audio_buffer.append",
+            "audio": base64.b64encode(audio).decode("ascii"),
+        }
+    )
+
+
+async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
+    await connection.send(
+        json.dumps(
+            {
+                "type": "session.update",
+                "session": {"audio": {"output": {"language": target_language}}},
+            }
+        )
+    )
+    while True:
+        raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
+        event = json.loads(raw_event)
+        event_type = event.get("type")
+        if event_type == "session.created":
+            write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
+        if event_type == "session.updated":
+            return None
+        if event_type == "error":
+            return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
+
+
+async def send_audio(
+    connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
+) -> None:
+    silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
+    try:
+        for chunk in audio_chunks(pcm + silence):
+            await connection.send(audio_message(chunk))
+            await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
+        await connection.send(json.dumps({"type": "session.close"}))
+    finally:
+        finished.set()
+
+
+async def receive_translation(
+    connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
+) -> str | None:
+    audio_received = asyncio.Event()
+    try:
+        with wave.open(str(output_path), "wb") as output:
+            output.setnchannels(CHANNELS)
+            output.setsampwidth(SAMPLE_WIDTH)
+            output.setframerate(SAMPLE_RATE)
+            write_stdout("Translation: ", end="", flush=True)
+            while True:
+                try:
+                    raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
+                except TimeoutError:
+                    if not sender_finished.is_set():
+                        continue
+                    return (
+                        f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
+                    )
+                event = json.loads(raw_event)
+                event_type = event.get("type")
+                if event_type in AUDIO_EVENT_TYPES:
+                    output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
+                    audio_received.set()
+                elif event_type in TRANSCRIPT_EVENT_TYPES:
+                    write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
+                elif event_type == "error":
+                    return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
+                elif event_type == "session.closed":
+                    write_stdout()
+                    return None if audio_received.is_set() else "The translation stream ended without translated audio"
+    except (OSError, wave.Error) as exc:
+        return f"Unable to write {output_path}: {exc}"
+
+
+async def translate(settings: Settings, pcm: bytes) -> str | None:
+    url = translation_url(settings.base_url, settings.model)
+    if not url:
+        return f"Invalid --base-url: {settings.base_url}"
+    sender_finished = asyncio.Event()
+    try:
+        async with websockets.connect(
+            url,
+            additional_headers={"Authorization": f"Bearer {settings.api_key}"},
+            proxy=None,
+            open_timeout=20,
+            close_timeout=5,
+        ) as connection:
+            configuration_error = await configure_session(connection, settings.target_language)
+            if configuration_error:
+                return configuration_error
+            async with asyncio.TaskGroup() as tasks:
+                receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
+                tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
+            return receiver.result()
+    except Exception as exc:
+        return f"Translation failed: {type(exc).__name__}: {exc}"
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    settings = parse_args(argv)
+    if isinstance(settings, str):
+        write_stderr(settings)
+        return 2
+    pcm = read_pcm16_wav(settings.input_wav)
+    if isinstance(pcm, str):
+        write_stderr(pcm)
+        return 2
+    error = asyncio.run(translate(settings, pcm))
+    if error:
+        write_stderr(error)
+        return 1
+    write_stdout(f"Translated audio: {settings.output_wav}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -3,9 +3,11 @@
 
 import asyncio
 import base64
+import copy
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from itertools import chain
 from types import MappingProxyType
@@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
 
 import litellm
 from litellm import Router, verbose_logger
-from litellm._internal_context import with_service_target
+from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.constants import (
@@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
     request_tags_from_metadata,
 )
 from litellm.repositories.managed_file_repository import ManagedFileRepository
+from litellm.responses.streaming_iterator import spawn_logging_task
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccessIssue]
     AllMessageValues,
@@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
+_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
+_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderUpload:
+    model_id: str
+    provider_file_id: str
+    upload_call_id: str | None = None
+
+
+def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
+    file_hidden_params: Final = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
+        dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
+    )
+    model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
+        file_hidden_params.get("model_file_id_mapping") or {}
+    )
+    upload_call_id: Final = file_hidden_params.get("litellm_call_id")
+    return tuple(
+        _ProviderUpload(
+            model_id=model_id,
+            provider_file_id=provider_file_id,
+            upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
+        )
+        for model_id, provider_file_id in model_file_id_mapping.items()
+    )
+
+
+def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
+    credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
+    return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
+
+
+def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
+    upload: Final = uploads[index]
+    other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
+    return all(other.model_id != upload.model_id for other in other_uploads)
+
+
+def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
+    trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
+    provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
+    return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
+
+
+def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
+    decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
+    carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
+    if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
+        return None
+    return carried
+
+
+async def _delete_provider_file(
+    llm_router: Router,
+    model_id: str,
+    provider_file_id: str,
+    request_data: Mapping[str, object],
+) -> None:
+    credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
+    delete_data: Final = {
+        **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
+        **(
+            {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
+            if credentials is not None
+            else {}
+        ),
+    }
+    await llm_router.afile_delete(
+        model=model_id,
+        file_id=provider_file_id,
+        **cast(_RouterFileCallKwargs, delete_data),
+    )
+
+
+def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
+    return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
+
+
+async def _delete_redundant_upload(
+    llm_router: Router,
+    upload: _ProviderUpload,
+    delete_request: Mapping[str, object],
+    parent_call_id: str | None,
+) -> None:
+    try:
+        with internal_sub_call(parent_call_id):
+            await _delete_provider_file(
+                llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
+            )
+    except Exception as error:
+        verbose_logger.warning(
+            f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
+            f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
+            f"{type(error).__name__} {sanitize_for_log(error)}"
+        )
+
+
+async def _delete_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    delete_request: Mapping[str, object],
+) -> None:
+    await asyncio.gather(
+        *(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
+    )
+
+
+def _start_deleting_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    create_file_request: CreateFileRequest,
+) -> None:
+    if not uploads:
+        return
+    delete_request: Final = MappingProxyType(
+        {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
+    )
+    with post_response_phase():
+        spawn_logging_task(
+            asyncio.get_running_loop(),
+            _delete_redundant_uploads(llm_router, uploads, delete_request),
+            task_name="managed file repeat upload cleanup",
+        )
+
+
 def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
     return prisma_client.db
 
@@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             target_model_names_list=target_model_names_list,
         )
 
-        ## STORE MODEL MAPPINGS IN DB
-        model_mappings: Dict[str, str] = {}
-
-        for file_object in responses:
-            file_hidden_params = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
-                dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
-            )
-            model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
-            if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
-                model_mappings.update(model_file_id_mapping)
+        uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
+        per_target_model_ids: Final = frozenset(
+            model_id
+            for model_id in {upload.model_id for upload in uploads}
+            if _keeps_one_upload_per_target(llm_router, model_id)
+        )
+        kept_uploads: Final = tuple(
+            upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
+        )
+        model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
 
         await self.store_unified_file_id(
             file_id=response.id,
@@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             model_mappings=model_mappings,
             user_api_key_dict=user_api_key_dict,
         )
+        _start_deleting_redundant_uploads(
+            llm_router,
+            tuple(
+                upload
+                for upload in uploads
+                if upload not in kept_uploads and upload.model_id not in per_target_model_ids
+            ),
+            create_file_request,
+        )
 
         # Emit Prometheus metrics for managed file creation
         prom_logger = self._get_prometheus_logger()
@@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
         if not specific_model_file_id_mapping:
             return
-        filtered_data: Final = {
-            k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
-        }
         for model_id, model_file_id in specific_model_file_id_mapping.items():
-            credentials = llm_router.registry().credentials(model_id, form="with_provider")
-            delete_data = {
-                **filtered_data,
-                **(
-                    {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
-                    if credentials is not None
-                    else {}
-                ),
-            }
-            await llm_router.afile_delete(
-                model=model_id,
-                file_id=model_file_id,
-                **cast(_RouterFileCallKwargs, delete_data),
+            await _delete_provider_file(llm_router, model_id, model_file_id, data)
+        unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
+        delete_call_id: Final = data.get("litellm_call_id")
+        if unrecorded_upload is not None:
+            await _delete_redundant_upload(
+                llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
             )
 
     async def afile_content(

litellm-rust/crates/callbacks-legacy-python/python_contract.json

@@ -33,6 +33,17 @@
     "api_key",
     "additional_args"
   ],
+  "record_response": [
+    "logger",
+    "status",
+    "headers",
+    "body"
+  ],
+  "record_api_window": [
+    "logger",
+    "start",
+    "end"
+  ],
   "defers_async_logging": [
     "logger"
   ],
@@ -91,6 +102,13 @@
     "logger",
     "head"
   ],
+  "stamp_stream_head": [
+    "head",
+    "logger",
+    "kwargs",
+    "start_time",
+    "end_time"
+  ],
   "stream_success": [
     "logger",
     "url_route",

litellm-rust/crates/callbacks-legacy-python/src/adapter.rs

@@ -57,6 +57,8 @@ pub struct LegacyLogging {
     asynchronous: bool,
     internal: bool,
     cache_key: Option<String>,
+    /// When the provider request went out, until its window is recorded (Messages only).
+    api_started: Option<f64>,
 }
 
 fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
@@ -87,6 +89,60 @@ impl LegacyLogging {
             asynchronous,
             internal: false,
             cache_key: None,
+            api_started: None,
+        }
+    }
+
+    /// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as
+    /// its own argument, so the logger never sees the caller's raw metadata, only the validated
+    /// request metadata merged in through `litellm_params`. Python updates the logger once, before
+    /// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the
+    /// logged metadata is not rebuilt from what the call wrote into it since.
+    fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult<Bound<'py, PyDict>> {
+        let kwargs = self.call.kwargs().bind(py).copy()?;
+        if self.operation != LoggingOperation::Messages {
+            return Ok(kwargs);
+        }
+        let consumed: &[&str] = if refresh {
+            &["metadata", "litellm_metadata"]
+        } else {
+            &["metadata"]
+        };
+        for name in consumed {
+            if kwargs.contains(name)? {
+                kwargs.del_item(name)?;
+            }
+        }
+        Ok(kwargs)
+    }
+
+    /// Python's Messages handler hands the logger its request params, validated `metadata`
+    /// included, as `litellm_params`, beside the `aanthropic_messages` call flag.
+    fn logged_request_params(
+        &self,
+        context: &RequestContext,
+    ) -> Option<serde_json::Map<String, Value>> {
+        if self.operation != LoggingOperation::Messages {
+            return None;
+        }
+        let mut params = context
+            .optional_params
+            .as_object()
+            .cloned()
+            .unwrap_or_default();
+        params.insert(
+            "aanthropic_messages".to_owned(),
+            Value::Bool(self.asynchronous),
+        );
+        Some(params)
+    }
+
+    /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync
+    /// client is untimed.
+    fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> {
+        match self.api_started.take() {
+            Some(start) => self.logger()?.record_api_window(py, start, end),
+            None => Ok(()),
         }
     }
 
@@ -111,6 +167,16 @@ impl LegacyLogging {
         }
     }
 
+    /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key.
+    fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> {
+        match self.operation {
+            LoggingOperation::Messages => Some(""),
+            LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => {
+                context.api_key.as_ref().map(|api_key| api_key.expose())
+            }
+        }
+    }
+
     fn stream_billing(&self) -> Option<PassThroughStream> {
         match self.operation {
             LoggingOperation::Messages => Some(PassThroughStream {
@@ -377,7 +443,13 @@ impl LegacyLogging {
         context: &RequestContext,
     ) -> PyResult<HookStep<Self, Box<WireRequest>>> {
         let logger = self.logger()?;
-        logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?;
+        logger.update_from_kwargs(
+            py,
+            &self.logging_kwargs(py, false)?,
+            &wire,
+            context,
+            self.logged_request_params(context).as_ref(),
+        )?;
         let body = to_py(py, &wire.body)?
             .into_bound(py)
             .cast_into::<PyDict>()?;
@@ -400,11 +472,14 @@ impl LegacyLogging {
         self.logger()?.pre_call(
             py,
             self.input_description(),
-            context.api_key.as_ref().map(|api_key| api_key.expose()),
+            self.logged_api_key(context),
             &body,
             &headers,
             &wire.url,
         )?;
+        if self.operation == LoggingOperation::Messages && self.asynchronous {
+            self.api_started = Some(epoch_seconds());
+        }
         let headers = headers
             .iter()
             .map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
@@ -455,7 +530,7 @@ impl LegacyLogging {
             py,
             (
                 &logger,
-                self.call.kwargs(),
+                self.logging_kwargs(py, true)?,
                 &facts.provider.model,
                 logger.getattr("optional_params")?,
                 params,
@@ -483,6 +558,16 @@ impl LegacyLogging {
         py: Python<'_>,
         raw: &RawResponse,
     ) -> PyResult<HookStep<Self, ()>> {
+        if self.operation == LoggingOperation::Messages {
+            // Python's Messages handler never calls `post_call` at receipt: it stores the raw
+            // response, and logging replays it through `AnthropicConfig.transform_response`.
+            self.close_api_window(py, epoch_seconds())?;
+            if let Some(head) = &raw.head {
+                self.logger()?
+                    .record_response(py, head.status, &head.headers, &raw.body)?;
+            }
+            return Ok(HookStep::Ready(()));
+        }
         let api_key = self
             .request
             .as_ref()
@@ -522,6 +607,9 @@ impl LegacyLogging {
     ) -> PyResult<HookStep<Self, ()>> {
         self.end = Some(datetime(py, timing.end_time)?);
         self.error = Some(error.clone_ref(py).into_value(py));
+        if self.logger.is_some() {
+            self.close_api_window(py, timing.end_time)?;
+        }
         if self.stream.is_some() {
             return self.stream_failure(py);
         }
@@ -543,6 +631,20 @@ impl LegacyLogging {
             head.bind(py).set_item("cache_key", key)?;
             head.bind(py).set_item("cache_hit", true)?;
         }
+        self.close_api_window(py, epoch_seconds())?;
+        if self.operation == LoggingOperation::Messages {
+            // Python's `@client` stamps the call metadata onto the stream it hands back.
+            Streaming::StampHead.call(
+                py,
+                (
+                    head.bind(py),
+                    self.logger()?.object(py),
+                    self.call.kwargs(),
+                    &self.start,
+                    datetime(py, epoch_seconds())?,
+                ),
+            )?;
+        }
         Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
         self.stream = Some(DeliveredStream {
             chunks: PyList::empty(py).unbind(),
@@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update
         let step = logging
             .before_provider_request(py, Box::new(wire), context)
             .unwrap();
-        let raw = RawResponse {
-            body: "raw response".into(),
-        };
+        let raw = RawResponse::new("raw response".into());
         assert!(matches!(
             logging
                 .on_event(
@@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response
                 py,
                 &locals,
                 c"
-assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
+assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls
 assert logger.calls[0][1] is head
-chunks = logger.calls[1][1]
+assert logger.calls[1][1] is head
+chunks = logger.calls[2][1]
 assert len(chunks) == 2
 assert chunks[0] is first
 assert chunks[1] is last

litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs

@@ -5,6 +5,7 @@
 use litellm_host::interceptors::{RequestContext, WireRequest};
 use litellm_host_python::to_py;
 use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
+use serde_json::Value;
 
 use crate::logger::PythonLogger;
 use crate::python::{Logging, Wrapper};
@@ -15,9 +16,10 @@ pub trait LegacyCallbacks {
     fn update_from_kwargs(
         &self,
         py: Python<'_>,
-        kwargs: &Py<PyDict>,
+        kwargs: &Bound<'_, PyDict>,
         wire: &WireRequest,
         context: &RequestContext,
+        request_params: Option<&serde_json::Map<String, Value>>,
     ) -> PyResult<()>;
 
     /// `Logging.pre_call`.
@@ -31,6 +33,20 @@ pub trait LegacyCallbacks {
         url: &str,
     ) -> PyResult<()>;
 
+    /// Records the provider's raw HTTP response on the logger, as Python's Messages handler
+    /// does with `httpx_response` and `response_headers`.
+    fn record_response(
+        &self,
+        py: Python<'_>,
+        status: u16,
+        headers: &[(String, String)],
+        body: &str,
+    ) -> PyResult<()>;
+
+    /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post`
+    /// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`.
+    fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>;
+
     /// `Logging.post_call`.
     fn post_call(
         &self,
@@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger {
     fn update_from_kwargs(
         &self,
         py: Python<'_>,
-        kwargs: &Py<PyDict>,
+        kwargs: &Bound<'_, PyDict>,
         wire: &WireRequest,
         context: &RequestContext,
+        request_params: Option<&serde_json::Map<String, Value>>,
     ) -> PyResult<()> {
         let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect();
-        let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?;
+        let redacted_kwargs = redact(py, kwargs, &secret_fields)?;
         let optional_params = redact(
             py,
             &to_py(py, &context.optional_params)?
@@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger {
             &secret_fields,
         )?;
         let params = PyDict::new(py);
-        params.set_item(
-            "litellm_call_id",
-            kwargs.bind(py).get_item("litellm_call_id")?,
-        )?;
+        params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
         params.set_item("api_base", &wire.url)?;
         for name in ["logger_fn", "litellm_request_debug"] {
-            if let Some(value) = kwargs.bind(py).get_item(name)? {
+            if let Some(value) = kwargs.get_item(name)? {
                 params.set_item(name, value)?;
             }
         }
         for name in custom_pricing_fields(py)? {
-            if let Some(value) = kwargs.bind(py).get_item(&name)?
+            if let Some(value) = kwargs.get_item(&name)?
                 && !value.is_none()
             {
                 params.set_item(name, value)?;
             }
         }
+        for (name, value) in request_params.into_iter().flatten() {
+            params.set_item(name, to_py(py, value)?)?;
+        }
         Logging::Update.call(
             py,
             (
@@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger {
         Ok(())
     }
 
+    fn record_response(
+        &self,
+        py: Python<'_>,
+        status: u16,
+        headers: &[(String, String)],
+        body: &str,
+    ) -> PyResult<()> {
+        Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?;
+        Ok(())
+    }
+
+    fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> {
+        Logging::RecordApiWindow.call(py, (self.object(py), start, end))?;
+        Ok(())
+    }
+
     fn post_call(
         &self,
         py: Python<'_>,

litellm-rust/crates/callbacks-legacy-python/src/python.rs

@@ -44,6 +44,10 @@ pub(crate) enum Logging {
     PreCall,
     #[strum(serialize = "post_call")]
     PostCall,
+    #[strum(serialize = "record_response")]
+    RecordResponse,
+    #[strum(serialize = "record_api_window")]
+    RecordApiWindow,
     #[strum(serialize = "defers_async_logging")]
     DefersAsync,
     #[strum(serialize = "defer_success")]
@@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks {
 pub(crate) enum Streaming {
     #[strum(serialize = "stream_opened")]
     Opened,
+    #[strum(serialize = "stamp_stream_head")]
+    StampHead,
     #[strum(serialize = "stream_success")]
     Success,
     #[strum(serialize = "stream_failure")]

litellm-rust/crates/callbacks-legacy-python/src/test_support.rs

@@ -65,6 +65,12 @@ FAKES = {
     'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call(
         original_response, api_key, additional_args
     ),
+    'record_response': lambda logger, status, headers, body: logger.model_call_details.update(
+        response_headers=dict(headers)
+    ),
+    'record_api_window': lambda logger, start, end: logger.model_call_details.update(
+        llm_api_duration_ms=(end - start) * 1000
+    ),
     'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)),
     'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending),
     'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls(
@@ -85,6 +91,7 @@ FAKES = {
     ),
     'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
     'stream_opened': lambda logger, head: logger.record('stream_opened', head),
+    'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head),
     'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
         'stream_success', list(chunks)
     ),

litellm-rust/crates/host-http/tests/serve.rs

@@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(Bytes::from(wire.url)))
         },
@@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once(
         move |_, _, route_hooks, _observations| async move {
             if event {
                 route_hooks
-                    .after_provider_response(RawResponse {
-                        body: "response".into(),
-                    })
+                    .after_provider_response(RawResponse::new("response".into()))
                     .await?;
             } else {
                 route_hooks
@@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(json!({"url": wire.url})))
         },
@@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting(
         None,
         |_, _, route_hooks, _observations| async move {
             route_hooks
-                .after_provider_response(RawResponse { body: "raw".into() })
+                .after_provider_response(RawResponse::new("raw".into()))
                 .await?;
             Err(TestError::Provider)
         },

litellm-rust/crates/host-native/tests/driver.rs

@@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine {
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(wire.url))
         },
@@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re
         "input",
         Some(sender),
         |_, _, interceptors, observers| async move {
-            let raw = RawResponse {
-                body: "provider response".into(),
-            };
+            let raw = RawResponse::new("provider response".into());
             observers.unwrap().emit(CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
             ));

litellm-rust/crates/host-python/tests/support/mod.rs

@@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory {
                     .before_provider_request(wire(), context())
                     .await?;
                 host.interceptors
-                    .after_provider_response(RawResponse { body: "raw".into() })
+                    .after_provider_response(RawResponse::new("raw".into()))
                     .await?;
                 Ok(format!("{projected}|{signed}|{}", wire.url))
             })

litellm-rust/crates/host/src/interceptors.rs

@@ -28,6 +28,28 @@ pub struct RequestContext {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct RawResponse {
     pub body: String,
+    /// The provider's HTTP status and headers, when the route forwards them to logging.
+    pub head: Option<ResponseHead>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ResponseHead {
+    pub status: u16,
+    pub headers: Vec<(String, String)>,
+}
+
+impl RawResponse {
+    pub fn new(body: String) -> Self {
+        Self { body, head: None }
+    }
+
+    #[must_use]
+    pub fn with_head(self, head: ResponseHead) -> Self {
+        Self {
+            head: Some(head),
+            ..self
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -186,7 +208,7 @@ mod tests {
                 .await?;
                 Interceptors::after_provider_response(
                     &channel.interceptors,
-                    RawResponse { body: "raw".into() },
+                    RawResponse::new("raw".into()),
                 )
                 .await?;
                 Ok((sent, ()))

litellm-rust/crates/host/tests/observation.rs

@@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile
 
 #[rstest]
 fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) {
-    let mut raw = RawResponse {
-        body: "provider response".into(),
-    };
+    let mut raw = RawResponse::new("provider response".into());
     let event: CallEvent<(), (), &RawResponse> =
         CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw });
     let expected = raw.clone();

litellm-rust/crates/http/src/error.rs

@@ -58,8 +58,20 @@ pub(crate) enum Repr {
     Network(#[source] reqwest::Error),
     #[error("upstream request timed out")]
     Timeout(#[source] reqwest::Error),
+    /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says
+    /// the response head had already arrived.
+    #[error("upstream request timed out")]
+    ReadTimeout {
+        #[source]
+        elapsed: Elapsed,
+        body: bool,
+    },
     #[error("upstream request failed with status {status}: {body}")]
-    Status { status: u16, body: UpstreamBody },
+    Status {
+        status: u16,
+        body: UpstreamBody,
+        headers: Vec<(String, String)>,
+    },
 
     #[error("invalid WebSocket URL")]
     WebSocketUrl(#[source] url::ParseError),
@@ -136,13 +148,59 @@ impl Repr {
 
 impl Error {
     pub fn upstream(status: u16, body: String) -> Self {
+        Self::upstream_with_headers(status, body, Vec::new())
+    }
+
+    /// An upstream failure that keeps the provider's response headers (`retry-after`,
+    /// `request-id`) for the caller.
+    pub fn upstream_with_headers(
+        status: u16,
+        body: String,
+        headers: Vec<(String, String)>,
+    ) -> Self {
         Repr::Status {
             status,
             body: UpstreamBody(body),
+            headers,
         }
         .into()
     }
 
+    /// The provider's response headers on an upstream failure; empty for every other error.
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match &*self.0 {
+            Repr::Status { headers, .. } => headers,
+            _ => &[],
+        }
+    }
+
+    /// A timeout reached before the connection was established, as opposed to while reading.
+    pub fn is_connect_timeout(&self) -> bool {
+        matches!(&*self.0, Repr::Timeout(error) if error.is_connect())
+    }
+
+    /// A read the caller timed itself (httpx's per-read timeout) ran out.
+    pub fn read_timeout(elapsed: Elapsed) -> Self {
+        Repr::ReadTimeout {
+            elapsed,
+            body: false,
+        }
+        .into()
+    }
+
+    /// A read of the response body the caller timed itself ran out, after the head arrived.
+    pub fn body_read_timeout(elapsed: Elapsed) -> Self {
+        Repr::ReadTimeout {
+            elapsed,
+            body: true,
+        }
+        .into()
+    }
+
+    pub fn is_body_read_timeout(&self) -> bool {
+        matches!(&*self.0, Repr::ReadTimeout { body: true, .. })
+    }
+
     pub fn computed_header(name: impl Into<String>) -> Self {
         Repr::ComputedHeader(name.into()).into()
     }
@@ -169,6 +227,7 @@ impl Error {
         match &*self.0 {
             Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::Handshake(_)
             | Repr::HandshakeTimeout(_)
@@ -227,6 +286,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::Handshake(_)
             | Repr::HandshakeTimeout(_)
@@ -259,6 +319,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -295,6 +356,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -333,6 +395,7 @@ impl Error {
             | Repr::Signature(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -356,7 +419,10 @@ impl Error {
 
     pub fn is_timeout(&self) -> bool {
         match &*self.0 {
-            Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true,
+            Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
+            | Repr::HandshakeTimeout(_)
+            | Repr::DownloadTimeout(_) => true,
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -392,7 +458,7 @@ impl Error {
 
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match &*self.0 {
-            Repr::Status { status, body } => Some((*status, &body.0)),
+            Repr::Status { status, body, .. } => Some((*status, &body.0)),
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -408,6 +474,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
             | Repr::WebSocketSchemeRewrite(_)
@@ -431,7 +498,7 @@ impl Error {
     pub fn status(&self) -> Option<u16> {
         match &*self.0 {
             Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status),
-            Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408),
+            Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408),
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -482,6 +549,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)

litellm-rust/crates/inference-chat/src/handler.rs

@@ -108,7 +108,7 @@ pub(super) async fn execute(
             if !status.is_success() {
                 return Err(litellm_http::Error::upstream(status.as_u16(), text).into());
             }
-            let raw = RawResponse { body: text.clone() };
+            let raw = RawResponse::new(text.clone());
             if let Some(observers) = observers {
                 observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                     ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference-messages/Cargo.toml

@@ -24,6 +24,7 @@ litellm-tracing.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
+tokio.workspace = true
 tracing.workspace = true
 
 [dev-dependencies]
@@ -34,5 +35,4 @@ litellm-host-native.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }
 litellm-inference-testing.workspace = true
 rstest.workspace = true
-tokio.workspace = true
 wiremock.workspace = true

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -35,6 +35,14 @@ impl MessagesProvider {
     pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
         self.config
     }
+
+    /// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped.
+    pub(crate) fn accepts_service_tier(self) -> bool {
+        matches!(
+            self.provider,
+            LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle
+        )
+    }
 }
 
 /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
@@ -119,3 +127,69 @@ mod tests {
         );
     }
 }
+
+/// Python's `_merge_messages_header_sources`: later sources win per header name, except
+/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and
+/// sorted into one lowercase `anthropic-beta` entry.
+pub fn merge_messages_header_sources(
+    sources: impl IntoIterator<Item = Map<String, Value>>,
+) -> Map<String, Value> {
+    let mut merged = Map::new();
+    let mut betas = std::collections::BTreeSet::new();
+    for (name, value) in sources.into_iter().flatten() {
+        if !name.eq_ignore_ascii_case("anthropic-beta") {
+            merged.insert(name, value);
+            continue;
+        }
+        let values: Vec<&str> = match &value {
+            Value::String(text) => vec![text.as_str()],
+            Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(),
+            _ => Vec::new(),
+        };
+        betas.extend(
+            values
+                .iter()
+                .flat_map(|value| value.split(','))
+                .map(str::trim)
+                .filter(|beta| !beta.is_empty())
+                .map(str::to_owned),
+        );
+    }
+    if !betas.is_empty() {
+        merged.insert(
+            "anthropic-beta".to_owned(),
+            Value::String(betas.into_iter().collect::<Vec<_>>().join(",")),
+        );
+    }
+    merged
+}
+
+#[cfg(test)]
+mod merge_tests {
+    use serde_json::{Map, Value, json};
+
+    use super::merge_messages_header_sources;
+
+    fn map(value: Value) -> Map<String, Value> {
+        value.as_object().cloned().unwrap()
+    }
+
+    #[test]
+    fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() {
+        let merged = merge_messages_header_sources([
+            map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})),
+            map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})),
+            map(json!({"anthropic-beta": 5})),
+        ]);
+        assert_eq!(
+            Value::Object(merged),
+            json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"})
+        );
+    }
+
+    #[test]
+    fn no_beta_values_leave_no_beta_header() {
+        let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]);
+        assert!(merged.is_empty());
+    }
+}

litellm-rust/crates/inference-messages/src/handler.rs

@@ -10,9 +10,9 @@ use litellm_core_utils::{
     get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
 };
 use litellm_host::interceptors::{
-    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
+    ResponseHead, ResultSource, WireRequest,
 };
-use litellm_http::request::with_default_headers;
 use litellm_llms::base_llm::{
     auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
@@ -21,14 +21,16 @@ use litellm_llms::base_llm::{
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
     Error, MessagesCall, MessagesCallResponse, MessagesRoute,
-    common_utils::{MessagesProvider, string_headers},
+    common_utils::{MessagesProvider, merge_messages_header_sources, string_headers},
     constants::MESSAGES_TIMEOUT_SECS,
     route::Messages,
     types::invalid_request,
@@ -126,14 +128,18 @@ async fn provider_call(
         MessagesRequest { model, ..body },
         shaping.settings.reasoning_auto_summary,
     )?;
+    let transform_context =
+        MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params);
+    let filtered = requested_optional_params(sanitized, provider, &transform_context)?;
     let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+        without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?;
 
     let scoped =
         get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
+    let forwarded = string_headers(Some(merge_messages_header_sources([
+        extra_headers.unwrap_or_default(),
+        scoped,
+    ])))?;
     let validated = config.validate_environment(
         forwarded,
         api_key.as_deref(),
@@ -144,16 +150,13 @@ async fn provider_call(
     )?;
 
     let body = config.request_body(
-        config.transform_anthropic_messages_request(
-            trimmed,
-            &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-        )?,
+        config.transform_anthropic_messages_request(trimmed, &transform_context)?,
         &validated.headers,
         shaping.settings.drop_params,
     )?;
     let environment = ValidatedEnvironment {
         headers: config.request_headers(
-            with_default_headers(validated.headers, config.default_headers()),
+            with_python_default_headers(validated.headers, config.default_headers()),
             &body,
         ),
         auth: validated.auth,
@@ -184,7 +187,7 @@ async fn provider_call(
         .before_provider_request(
             WireRequest {
                 url,
-                headers: authenticated.headers,
+                headers: joined_header_values(authenticated.headers),
                 body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
             request_context,
@@ -213,6 +216,64 @@ async fn provider_call(
     })
 }
 
+/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in
+/// case into one lowercase header whose values are joined with `", "`.
+fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> {
+    let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len());
+    let mut positions: std::collections::HashMap<String, usize> =
+        std::collections::HashMap::with_capacity(headers.len());
+    for (name, value) in headers {
+        match positions.entry(name.to_ascii_lowercase()) {
+            std::collections::hash_map::Entry::Occupied(position) => {
+                let present = &mut joined[*position.get()].1;
+                present.push_str(", ");
+                present.push_str(&value);
+            }
+            std::collections::hash_map::Entry::Vacant(position) => {
+                position.insert(joined.len());
+                joined.push((name, value));
+            }
+        }
+    }
+    joined
+}
+
+/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is
+/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it.
+fn with_python_default_headers(
+    headers: Vec<(String, String)>,
+    defaults: &[(&str, &str)],
+) -> Vec<(String, String)> {
+    let missing: Vec<(String, String)> = defaults
+        .iter()
+        .filter(|(name, _)| !headers.iter().any(|(present, _)| present == name))
+        .map(|(name, value)| ((*name).to_string(), (*value).to_string()))
+        .collect();
+    headers.into_iter().chain(missing).collect()
+}
+
+/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier
+/// and sampling params before `additional_drop_params` and the environment check run.
+fn requested_optional_params(
+    request: MessagesRequest,
+    provider: MessagesProvider,
+    context: &MessagesTransformContext,
+) -> Result<MessagesRequest, Error> {
+    let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params(
+        request, context,
+    )?;
+    if provider.accepts_service_tier() {
+        return Ok(request);
+    }
+    Ok(MessagesRequest {
+        params: MessagesOptionalParams {
+            service_tier: None,
+            ..request.params
+        },
+        ..request
+    })
+}
+
 fn without_additional_drop_params(
     request: MessagesRequest,
     paths: &[String],
@@ -245,6 +306,7 @@ async fn call_provider(
     } = request;
     let provider_name = provider.as_str();
     log_request_body(provider_name, stream, &wire.body);
+    let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
     let response = send(
         http,
         Authenticated {
@@ -254,10 +316,11 @@ async fn call_provider(
         &wire.url,
         &wire.body,
         timeout,
+        stream,
     )
     .await?;
     if !response.status().is_success() {
-        return Err(provider_error(response).await);
+        return Err(provider_error(response, timeout).await);
     }
     let config = provider.config();
     if stream {
@@ -265,11 +328,24 @@ async fn call_provider(
             response,
             config.stream_decoder(),
             provider_name,
+            timeout,
         ));
     }
+    let head = ResponseHead {
+        status: response.status().as_u16(),
+        headers: litellm_http::request::header_pairs(response.headers()),
+    };
+    let status = head.status;
     let text = response.text().await.map_err(litellm_http::Error::from)?;
     log_response_body(&text);
-    context.response_received(&text).await?;
+    context
+        .raw_response_received(RawResponse::new(text.clone()).with_head(head))
+        .await?;
+    if serde_json::from_str::<serde::de::IgnoredAny>(&text).is_err() {
+        // Python raises `AnthropicError(status_code=<the response's>, message=<its text>)` for a
+        // body it cannot parse, and the public mapping classifies it by that status.
+        return Err(litellm_http::Error::upstream(status, text).into());
+    }
     decode_response(config, &identity.model, &text)
         .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
@@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error {
     ))
 }
 
+/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete
+/// response keeps the client's overall deadline; a stream is bounded until its head arrives and
+/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off.
 async fn send(
     http: &litellm_http::Client,
     authenticated: Authenticated,
     url: &str,
     body: &Value,
-    timeout: Option<Duration>,
+    timeout: Duration,
+    stream: bool,
 ) -> Result<reqwest::Response, Error> {
-    let request = outbound_request(
-        authenticated,
-        url,
-        body,
-        Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))),
-    )?;
-    Ok(litellm_inference::outbound::send(request, http).await?)
+    let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?;
+    if !stream {
+        return Ok(litellm_inference::outbound::send(request, http).await?);
+    }
+    tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http))
+        .await
+        .map_err(litellm_http::Error::read_timeout)?
+        .map_err(Error::from)
 }
 
-async fn provider_error(response: reqwest::Response) -> Error {
+/// The stream's body, each read bounded by `timeout` as httpx bounds each read.
+fn timed_chunks(
+    response: reqwest::Response,
+    timeout: Duration,
+) -> impl futures_util::Stream<Item = Result<Bytes, litellm_http::Error>> + Send + 'static {
+    futures_util::stream::try_unfold(response, move |mut response| async move {
+        let chunk = tokio::time::timeout(timeout, response.chunk())
+            .await
+            .map_err(litellm_http::Error::body_read_timeout)?
+            .map_err(litellm_http::Error::from)?;
+        Ok(chunk.map(|chunk| (chunk, response)))
+    })
+}
+
+/// The provider's error, its body read under `timeout`: a stream has no overall deadline once
+/// its head arrives, so an error body that stalls would otherwise hold the call forever.
+async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error {
     let status = response.status().as_u16();
-    match response.text().await {
-        Ok(text) => {
+    let headers = litellm_http::request::header_pairs(response.headers());
+    match tokio::time::timeout(timeout, response.text()).await {
+        Ok(Ok(text)) => {
             log_error_body(status, &text);
-            litellm_http::Error::upstream(status, text).into()
+            litellm_http::Error::upstream_with_headers(status, text, headers).into()
         }
-        Err(error) => litellm_http::Error::from(error).into(),
+        Ok(Err(error)) => litellm_http::Error::from(error).into(),
+        Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(),
     }
 }
 
@@ -329,22 +428,22 @@ fn streaming_response(
     response: reqwest::Response,
     decoder: Option<StreamDecoder>,
     provider: &'static str,
+    timeout: Duration,
 ) -> MessagesCallResponse {
     let headers = response
         .headers()
         .iter()
         .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string())))
         .collect();
+    let chunks = timed_chunks(response, timeout)
+        .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
     let chunks = match decoder {
-        None => futures_util::stream::try_unfold(response, move |mut response| async move {
-            let chunk = response.chunk().await.map_err(litellm_http::Error::from)?;
-            Ok(chunk.map(|chunk| {
-                log_chunk(provider, "provider_response", &chunk);
-                (chunk, response)
-            }))
-        })
-        .boxed(),
-        Some(decode) => decoded_chunks(response, decode, provider),
+        None => chunks.map_err(Error::from).boxed(),
+        Some(decode) => decoded_chunks(
+            chunks.map_err(std::io::Error::other).boxed(),
+            decode,
+            provider,
+        ),
     };
     MessagesCallResponse::Stream {
         head: super::route::MessagesStreamHead { headers },
@@ -353,15 +452,10 @@ fn streaming_response(
 }
 
 fn decoded_chunks(
-    response: reqwest::Response,
+    bytes: ByteStream,
     decode: StreamDecoder,
     provider: &'static str,
 ) -> BoxStream<'static, Result<Bytes, Error>> {
-    let bytes: ByteStream = response
-        .bytes_stream()
-        .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk))
-        .map_err(|error| std::io::Error::other(litellm_http::Error::from(error)))
-        .boxed();
     futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
         let Some(event) = events.try_next().await? else {
             return Ok(None);
@@ -964,9 +1058,12 @@ mod tests {
             .send()
             .await
             .unwrap();
-        let MessagesCallResponse::Stream { mut chunks, .. } =
-            streaming_response(response, Some(streaming::event_stream), "test")
-        else {
+        let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
+            response,
+            Some(streaming::event_stream),
+            "test",
+            Duration::from_secs(5),
+        ) else {
             panic!("a streaming response returns chunks");
         };
 

litellm-rust/crates/inference-messages/src/lib.rs

@@ -14,6 +14,7 @@ use std::sync::Arc;
 
 use crate::common_utils::messages_provider;
 
+pub use common_utils::merge_messages_header_sources;
 pub use litellm_inference::Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
 #[case::deepseek("deepseek")]
 #[case::minimax("minimax")]
 #[tokio::test]
-async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
+/// Python adds a default unless that exact lowercase name is present, then folds the casings
+/// into one header, so a capitalised caller header is sent beside the default.
+async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults(
+    call: MessagesCall,
+    #[case] provider: &str,
+) {
     let upstream = upstream([message_response()]).await;
 
     run_message(MessagesCall {
@@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
     .await;
 
     let request = only_request(&upstream).await;
-    assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]);
+    assert_eq!(
+        request.header_values("anthropic-version"),
+        ["2024-01-01, 2023-06-01"]
+    );
     assert_eq!(
         request.header_values("content-type"),
-        ["application/json; charset=utf-8"]
+        ["application/json; charset=utf-8, application/json"]
     );
 }
 

litellm-rust/crates/inference-messages/tests/messages/response.rs

@@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case]
 }
 
 #[rstest]
-#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))]
-#[case::not_a_message(json_response(json!({"unexpected": true})))]
 #[tokio::test]
-async fn an_unreadable_success_body_is_an_invalid_response(
-    call: MessagesCall,
-    #[case] response: ResponseTemplate,
-) {
-    let upstream = upstream([response]).await;
+async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) {
+    let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await;
+
+    let error = run(MessagesCall {
+        api_key: Some("sk".into()),
+        api_base: Some(upstream.uri()),
+        ..call
+    })
+    .await
+    .expect_err("an unparseable body fails");
+
+    assert_eq!(error.upstream_response(), Some((200, "not json")));
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) {
+    let upstream = upstream([json_response(json!({"unexpected": true}))]).await;
 
     let error = run(MessagesCall {
         api_key: Some("sk".into()),

litellm-rust/crates/inference-messages/tests/messages/stream.rs

@@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream(
         .unwrap();
 }
 
+/// Answers with an error head whose promised body never arrives.
+async fn stalling_error_upstream() -> (String, JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let base = format!("http://{}", listener.local_addr().unwrap());
+    let connection = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.unwrap();
+        let mut request = vec![0; 4096];
+        let _ = socket.read(&mut request).await;
+        socket
+            .write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n")
+            .await
+            .unwrap();
+        let _ = socket.read_to_end(&mut Vec::new()).await;
+    });
+    (base, connection)
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) {
+    let (base, connection) = stalling_error_upstream().await;
+    let host = RecordingStreamHost::new(
+        MessagesCall {
+            timeout: Some(Duration::from_millis(300)),
+            ..streaming(call, base)
+        },
+        usize::MAX,
+    );
+
+    let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host))
+        .await
+        .expect("the stalled error body gives up within the timeout")
+        .expect_err("a stalled error body fails the call");
+
+    assert!(error.dispatched(), "{error:?}");
+    assert_eq!(error.upstream_response(), None);
+    assert!(host.seen.into_inner().unwrap().is_empty());
+    tokio::time::timeout(Duration::from_secs(5), connection)
+        .await
+        .expect("timing out closes the upstream connection")
+        .unwrap();
+}
+
 #[rstest]
 #[tokio::test]
 async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) {

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -91,9 +91,7 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
     }
 
     fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-        let raw = RawResponse {
-            body: String::from_utf8_lossy(body).into_owned(),
-        };
+        let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned());
         if let Some(observers) = self.observers {
             observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference-responses/src/handler.rs

@@ -114,7 +114,7 @@ pub(super) async fn execute(
                 });
             }
             let body = response.text().await.map_err(litellm_http::Error::from)?;
-            let raw = RawResponse { body: body.clone() };
+            let raw = RawResponse::new(body.clone());
             if let Some(observers) = observers {
                 observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                     ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference/src/context.rs

@@ -42,9 +42,11 @@ impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
     }
 
     pub async fn response_received(&self, body: &str) -> Result<(), Error> {
-        let raw = RawResponse {
-            body: body.to_owned(),
-        };
+        self.raw_response_received(RawResponse::new(body.to_owned()))
+            .await
+    }
+
+    pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> {
         if let Some(observers) = &self.observers {
             observers.emit(CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference/src/error.rs

@@ -146,6 +146,34 @@ impl Error {
         }
     }
 
+    /// The provider's response headers on an upstream failure; empty for every other error.
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match self {
+            Self::Llm(error) => error.upstream_headers(),
+            Self::Http(error) => error.upstream_headers(),
+            _ => &[],
+        }
+    }
+
+    /// A read of a streamed body timed out after the response head arrived.
+    pub fn body_read_timeout(&self) -> bool {
+        match self {
+            Self::Llm(error) => error.is_body_read_timeout(),
+            Self::Http(error) => error.is_body_read_timeout(),
+            _ => false,
+        }
+    }
+
+    /// `Some(connect)` when the call timed out, where `connect` says the connection was never
+    /// established; `None` for every other error.
+    pub fn timeout(&self) -> Option<bool> {
+        match self {
+            Self::Llm(error) => error.is_timeout(),
+            Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+            _ => None,
+        }
+    }
+
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match self {
             Self::Llm(error) => error.upstream_response(),

litellm-rust/crates/llms-types/src/formats/messages/request.rs

@@ -264,7 +264,12 @@ pub enum ThinkingDisplay {
 #[macro_rules_attribute::apply(crate::wire_type)]
 #[derive(Default)]
 pub struct EnabledThinking {
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    /// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0.
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "crate::recognized::present"
+    )]
     pub budget_tokens: Option<Recognized<u64>>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub display: Option<Recognized<ThinkingDisplay>>,

litellm-rust/crates/llms-types/src/formats/messages/response.rs

@@ -10,14 +10,29 @@ pub struct MessagesResponse {
     pub content: Vec<Value>,
     pub stop_reason: Option<String>,
     pub stop_sequence: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    /// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps.
+    #[serde(
+        default,
+        deserialize_with = "present_value",
+        skip_serializing_if = "Option::is_none"
+    )]
     pub usage: Option<Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(
+        default,
+        deserialize_with = "present_value",
+        skip_serializing_if = "Option::is_none"
+    )]
     pub container: Option<Value>,
     #[serde(flatten)]
     pub extra: Map<String, Value>,
 }
 
+fn present_value<'de, D: serde::Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<Value>, D::Error> {
+    <Value as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -73,4 +88,21 @@ mod tests {
         assert_eq!(body.get("usage").cloned(), usage);
         assert_eq!(body.get("container").cloned(), container);
     }
+
+    #[test]
+    fn explicit_null_usage_and_container_survive_a_round_trip() {
+        let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
+            "content": [], "stop_reason": null, "stop_sequence": null,
+            "usage": null, "container": null});
+        let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
+        assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
+    }
+
+    #[test]
+    fn absent_usage_and_container_stay_absent() {
+        let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
+            "content": [], "stop_reason": null, "stop_sequence": null});
+        let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
+        assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
+    }
 }

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -124,12 +124,86 @@ pub enum AnthropicBeta {
     ToolSearchTool20251019,
     #[strum(serialize = "inline-tools-2026-09-15")]
     InlineTools20260915,
+    #[strum(serialize = "afk-mode-2026-01-31")]
+    AfkMode20260131,
+    #[strum(serialize = "agent-memory-2026-07-22")]
+    AgentMemory20260722,
+    #[strum(serialize = "analytics-usage-cost-2026-08-24")]
+    AnalyticsUsageCost20260824,
+    #[strum(serialize = "cache-diagnosis-2026-04-07")]
+    CacheDiagnosis20260407,
+    #[strum(serialize = "ce-plugins-2026-09-01")]
+    CePlugins20260901,
+    #[strum(serialize = "ce-user-management-2026-07-13")]
+    CeUserManagement20260713,
+    #[strum(serialize = "code-execution-2025-05-22")]
+    CodeExecution20250522,
+    #[strum(serialize = "computer-use-2024-10-22")]
+    ComputerUse20241022,
+    #[strum(serialize = "dev-full-thinking-2025-05-14")]
+    DevFullThinking20250514,
+    #[strum(serialize = "dreaming-2026-04-21")]
+    Dreaming20260421,
+    #[strum(serialize = "experimental-claude-tag-2026-10-21")]
+    ExperimentalClaudeTag20261021,
+    #[strum(serialize = "extended-cache-ttl-2025-04-11")]
+    ExtendedCacheTtl20250411,
+    #[strum(serialize = "fallback-credit-2026-06-01")]
+    FallbackCredit20260601,
+    #[strum(serialize = "fallback-credit-2026-07-01")]
+    FallbackCredit20260701,
+    #[strum(serialize = "managed-agents-2026-04-01")]
+    ManagedAgents20260401,
+    #[strum(serialize = "mcp-client-2026-09-15")]
+    McpClient20260915,
+    #[strum(serialize = "mcp-tunnels-2026-06-22")]
+    McpTunnels20260622,
+    #[strum(serialize = "message-batches-2024-09-24")]
+    MessageBatches20240924,
+    #[strum(serialize = "message-threads-2026-08-12")]
+    MessageThreads20260812,
+    #[strum(serialize = "mid-conversation-system-clear-at-2026-08-21")]
+    MidConversationSystemClearAt20260821,
+    #[strum(serialize = "model-context-window-exceeded-2025-08-26")]
+    ModelContextWindowExceeded20250826,
+    #[strum(serialize = "output-300k-2026-03-24")]
+    Output300k20260324,
+    #[strum(serialize = "pdfs-2024-09-25")]
+    Pdfs20240925,
+    #[strum(serialize = "prompt-caching-2024-07-31")]
+    PromptCaching20240731,
+    #[strum(serialize = "redact-thinking-2026-02-12")]
+    RedactThinking20260212,
+    #[strum(serialize = "server-side-fallback-2026-06-01")]
+    ServerSideFallback20260601,
+    #[strum(serialize = "server-side-fallback-2026-07-01")]
+    ServerSideFallback20260701,
+    #[strum(serialize = "spend-limit-reads-2026-09-26")]
+    SpendLimitReads20260926,
+    #[strum(serialize = "structured-outputs-2025-12-15")]
+    StructuredOutputs20251215,
+    #[strum(serialize = "task-budgets-2026-03-13")]
+    TaskBudgets20260313,
+    #[strum(serialize = "telemetry-destinations-2026-08-11")]
+    TelemetryDestinations20260811,
+    #[strum(serialize = "thinking-token-count-2026-05-13")]
+    ThinkingTokenCount20260513,
+    #[strum(serialize = "token-counting-2024-11-01")]
+    TokenCounting20241101,
+    #[strum(serialize = "user-profiles-2026-03-24")]
+    UserProfiles20260324,
+    #[strum(serialize = "user-profiles-2026-08-18")]
+    UserProfiles20260818,
+    #[strum(serialize = "user-profiles-2026-09-04")]
+    UserProfiles20260904,
+    #[strum(serialize = "webhooks-2026-10-07")]
+    Webhooks20261007,
     #[strum(default, transparent)]
     Other(String),
 }
 
 impl AnthropicBeta {
-    pub const KNOWN: [Self; 40] = [
+    pub const KNOWN: [Self; 77] = [
         Self::Oauth20250420,
         Self::WebFetch20250910,
         Self::WebSearch20250305,
@@ -170,6 +244,43 @@ impl AnthropicBeta {
         Self::ToolExamples20251029,
         Self::ToolSearchTool20251019,
         Self::InlineTools20260915,
+        Self::AfkMode20260131,
+        Self::AgentMemory20260722,
+        Self::AnalyticsUsageCost20260824,
+        Self::CacheDiagnosis20260407,
+        Self::CePlugins20260901,
+        Self::CeUserManagement20260713,
+        Self::CodeExecution20250522,
+        Self::ComputerUse20241022,
+        Self::DevFullThinking20250514,
+        Self::Dreaming20260421,
+        Self::ExperimentalClaudeTag20261021,
+        Self::ExtendedCacheTtl20250411,
+        Self::FallbackCredit20260601,
+        Self::FallbackCredit20260701,
+        Self::ManagedAgents20260401,
+        Self::McpClient20260915,
+        Self::McpTunnels20260622,
+        Self::MessageBatches20240924,
+        Self::MessageThreads20260812,
+        Self::MidConversationSystemClearAt20260821,
+        Self::ModelContextWindowExceeded20250826,
+        Self::Output300k20260324,
+        Self::Pdfs20240925,
+        Self::PromptCaching20240731,
+        Self::RedactThinking20260212,
+        Self::ServerSideFallback20260601,
+        Self::ServerSideFallback20260701,
+        Self::SpendLimitReads20260926,
+        Self::StructuredOutputs20251215,
+        Self::TaskBudgets20260313,
+        Self::TelemetryDestinations20260811,
+        Self::ThinkingTokenCount20260513,
+        Self::TokenCounting20241101,
+        Self::UserProfiles20260324,
+        Self::UserProfiles20260818,
+        Self::UserProfiles20260904,
+        Self::Webhooks20261007,
     ];
 
     pub fn as_str(&self) -> &str {
@@ -195,10 +306,9 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 => {
+            Self::AdvancedToolUse20251120 | Self::ToolSearchTool20251019 => {
                 &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
-            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -207,98 +317,99 @@ impl AnthropicBeta {
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260904 => &[
+            Self::AfkMode20260131
+            | Self::CePlugins20260901
+            | Self::CeUserManagement20260713
+            | Self::Compact20260904
+            | Self::Dreaming20260421
+            | Self::ExperimentalClaudeTag20261021
+            | Self::ExtendedCacheTtl20250411
+            | Self::FallbackCredit20260601
+            | Self::FallbackCredit20260701
+            | Self::InlineTools20260915
+            | Self::ManagedAgents20260401
+            | Self::McpTunnels20260622
+            | Self::MidConversationSystemClearAt20260821
+            | Self::ModelContextWindowExceeded20250826
+            | Self::Oauth20250420
+            | Self::Pdfs20240925
+            | Self::PromptCaching20240731
+            | Self::PromptCachingScope20260105
+            | Self::RedactThinking20260212
+            | Self::ServerSideFallback20260601
+            | Self::SpendLimitReads20260926
+            | Self::StructuredOutputs20251215
+            | Self::TaskBudgets20260313
+            | Self::TelemetryDestinations20260811
+            | Self::ThinkingTokenCount20260513
+            | Self::TokenCounting20241101
+            | Self::UserProfiles20260324
+            | Self::UserProfiles20260818
+            | Self::UserProfiles20260904 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Bash20241022
+            Self::AgentMemory20260722
+            | Self::Bash20241022
             | Self::Bash20250124
+            | Self::DevFullThinking20250514
             | Self::McpServers20251204
             | Self::StructuredOutput20240301
             | Self::TextEditor20241022
             | Self::TextEditor20250124 => BetaProvider::VARIANTS,
-            Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::CodeExecution20250825
+            Self::AnalyticsUsageCost20260824
+            | Self::CacheDiagnosis20260407
+            | Self::CodeExecution20250522
+            | Self::CodeExecution20250825
+            | Self::ComputerUse20241022
+            | Self::FastMode20260201
             | Self::FilesApi20250414
             | Self::McpClient20250404
             | Self::McpClient20251120
-            | Self::PromptCachingScope20260105
+            | Self::McpClient20260915
+            | Self::MessageBatches20240924
+            | Self::MessageThreads20260812
+            | Self::Output300k20260324
+            | Self::ServerSideFallback20260701
             | Self::Skills20251002
-            | Self::WebFetch20250910 => &[
+            | Self::WebFetch20250910
+            | Self::Webhooks20261007 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
+            Self::ClaudeCode20250219
+            | Self::PerTurnControl20260701
+            | Self::ToolExamples20251029 => &[
+                BetaProvider::BedrockConverse,
+                BetaProvider::Bedrock,
+                BetaProvider::Databricks,
+            ],
+            Self::Compact20260112
+            | Self::ContextManagement20250627
+            | Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
+            | Self::Effort20251124
+            | Self::FineGrainedToolStreaming20250514
             | Self::ThinkingBindingControls20260801
             | Self::ThinkingDisplayUpdates20260818 => &[],
-            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
-            Self::Effort20251124 => &[BetaProvider::VertexAi],
-            Self::FastMode20260201 | Self::Oauth20250420 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::FineGrainedToolStreaming20250514 => {
-                &[BetaProvider::AzureAi, BetaProvider::VertexAi]
-            }
-            Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
-                &[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
-            }
-            Self::MidConversationOutputConfig20260701 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::MidConversationToolChanges20260701 => &[
+            Self::InterleavedThinking20250514
+            | Self::Output128k20250219
+            | Self::TokenEfficientTools20250219
+            | Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock],
+            Self::MidConversationOutputConfig20260701
+            | Self::MidConversationToolChanges20260701 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-            ],
-            Self::PerTurnControl20260701 => &[
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::Databricks,
-            ],
-            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ToolSearchTool20251019 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Databricks,
-            ],
-            Self::InlineTools20260915 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::Databricks,
-            ],
+            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock],
             Self::Other(_) => BetaProvider::VARIANTS,
         }
     }
@@ -451,7 +562,44 @@ mod tests {
             AnthropicBeta::TokenEfficientTools20250219,
             AnthropicBeta::ToolExamples20251029,
             AnthropicBeta::ToolSearchTool20251019,
-            AnthropicBeta::InlineTools20260915
+            AnthropicBeta::InlineTools20260915,
+            AnthropicBeta::AfkMode20260131,
+            AnthropicBeta::AgentMemory20260722,
+            AnthropicBeta::AnalyticsUsageCost20260824,
+            AnthropicBeta::CacheDiagnosis20260407,
+            AnthropicBeta::CePlugins20260901,
+            AnthropicBeta::CeUserManagement20260713,
+            AnthropicBeta::CodeExecution20250522,
+            AnthropicBeta::ComputerUse20241022,
+            AnthropicBeta::DevFullThinking20250514,
+            AnthropicBeta::Dreaming20260421,
+            AnthropicBeta::ExperimentalClaudeTag20261021,
+            AnthropicBeta::ExtendedCacheTtl20250411,
+            AnthropicBeta::FallbackCredit20260601,
+            AnthropicBeta::FallbackCredit20260701,
+            AnthropicBeta::ManagedAgents20260401,
+            AnthropicBeta::McpClient20260915,
+            AnthropicBeta::McpTunnels20260622,
+            AnthropicBeta::MessageBatches20240924,
+            AnthropicBeta::MessageThreads20260812,
+            AnthropicBeta::MidConversationSystemClearAt20260821,
+            AnthropicBeta::ModelContextWindowExceeded20250826,
+            AnthropicBeta::Output300k20260324,
+            AnthropicBeta::Pdfs20240925,
+            AnthropicBeta::PromptCaching20240731,
+            AnthropicBeta::RedactThinking20260212,
+            AnthropicBeta::ServerSideFallback20260601,
+            AnthropicBeta::ServerSideFallback20260701,
+            AnthropicBeta::SpendLimitReads20260926,
+            AnthropicBeta::StructuredOutputs20251215,
+            AnthropicBeta::TaskBudgets20260313,
+            AnthropicBeta::TelemetryDestinations20260811,
+            AnthropicBeta::ThinkingTokenCount20260513,
+            AnthropicBeta::TokenCounting20241101,
+            AnthropicBeta::UserProfiles20260324,
+            AnthropicBeta::UserProfiles20260818,
+            AnthropicBeta::UserProfiles20260904,
+            AnthropicBeta::Webhooks20261007
         )]
         beta: AnthropicBeta,
     ) {
@@ -551,7 +699,7 @@ mod tests {
         BetaProvider::Anthropic,
         Some(AnthropicBeta::Oauth20250420)
     )]
-    #[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
+    #[case::rejected("skills-2025-10-02", BetaProvider::VertexAi, None)]
     #[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
     fn on_resolves_known_spellings_held_as_other(
         #[case] raw: &str,
@@ -580,12 +728,11 @@ mod tests {
 
     #[test]
     fn unknown_values_are_kept_verbatim() {
-        let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();
-        assert_eq!(
-            parsed,
-            AnthropicBeta::Other("claude-code-20250219".to_string())
+        let parsed: AnthropicBeta = "example-beta-2099-01-01".parse().unwrap();
+        assert!(
+            matches!(&parsed, AnthropicBeta::Other(value) if value == "example-beta-2099-01-01")
         );
-        assert_eq!(parsed.to_string(), "claude-code-20250219");
+        assert_eq!(parsed.to_string(), "example-beta-2099-01-01");
     }
 
     #[test]

litellm-rust/crates/llms-types/src/recognized.rs

@@ -7,6 +7,16 @@ pub enum Recognized<T> {
     Unrecognized(Value),
 }
 
+/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay
+/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`.
+pub fn present<'de, D, T>(deserializer: D) -> Result<Option<Recognized<T>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    Recognized<T>: serde::Deserialize<'de>,
+{
+    <Recognized<T> as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
+}
+
 impl<T> Recognized<T> {
     pub fn known(&self) -> Option<&T> {
         match self {

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -96,7 +96,10 @@ mod tests {
     }
 
     #[rstest]
-    #[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
+    #[case::azure_ai(
+        BetaProvider::AzureAi,
+        "advisor-tool-2026-03-01,example-beta-2099-01-01"
+    )]
     #[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
     #[case::vertex_ai(BetaProvider::VertexAi, " , ")]
     fn filter_removes_header_when_nothing_survives(

litellm-rust/crates/llms/src/anthropic/common_utils.rs

@@ -154,11 +154,42 @@ pub fn complete_anthropic_url(
 ) -> String {
     let api_base = resolve_anthropic_api_base(api_base, env_lookup);
 
-    let api_base = api_base.trim_end_matches('/');
-    if api_base.ends_with(MESSAGES_PATH) {
+    ensure_api_route(&api_base, MESSAGES_PATH)
+}
+
+/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds.
+fn url_path(url: &str) -> &str {
+    let before_fragment = url.split('#').next().unwrap_or_default();
+    let before_query = before_fragment.split('?').next().unwrap_or_default();
+    match before_query.split_once("://") {
+        Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]),
+        None => before_query,
+    }
+}
+
+/// Python's `_strip_trailing_slashes_keeping_the_scheme`.
+fn strip_trailing_slashes(api_base: &str) -> String {
+    let stripped = api_base.trim_end_matches('/');
+    if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') {
         return api_base.to_string();
     }
-    format!("{api_base}{MESSAGES_PATH}")
+    match stripped.split_once('?') {
+        Some((before, query)) if url_path(stripped).ends_with('/') => {
+            format!("{}?{query}", before.trim_end_matches('/'))
+        }
+        _ => stripped.to_string(),
+    }
+}
+
+/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as
+/// it is, query string included; anything else gets the route appended to the whole string.
+fn ensure_api_route(api_base: &str, route: &str) -> String {
+    let base = strip_trailing_slashes(api_base);
+    let route = route.trim_start_matches('/');
+    if url_path(&base).ends_with(&format!("/{route}")) {
+        return base;
+    }
+    format!("{}/{route}", strip_trailing_slashes(&base))
 }
 
 pub fn existing_betas(headers: &[(String, String)]) -> BetaSet {
@@ -1922,4 +1953,24 @@ mod tests {
             expected
         );
     }
+
+    #[rstest]
+    #[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")]
+    #[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")]
+    #[case::route_present(
+        "https://api.example.com/v1/messages/",
+        "https://api.example.com/v1/messages"
+    )]
+    #[case::route_and_query(
+        "https://api.example.com/v1/messages?beta=true",
+        "https://api.example.com/v1/messages?beta=true"
+    )]
+    #[case::query_without_route(
+        "https://api.example.com/?x=1",
+        "https://api.example.com?x=1/v1/messages"
+    )]
+    #[case::scheme_only("https://", "https:///v1/messages")]
+    fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) {
+        assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url);
+    }
 }

litellm-rust/crates/llms/src/anthropic/messages/thinking.rs

@@ -2,8 +2,8 @@ use litellm_llms_types::{
     formats::{
         chat_completions::ReasoningEffort,
         messages::{
-            EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig,
-            ThinkingDisplay,
+            AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig,
+            ThinkingConfig, ThinkingDisplay,
         },
     },
     recognized::Recognized,
@@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti
 
 fn effort_for_budget(
     budgets: &ThinkingBudgets,
-    budget_tokens: u64,
+    budget_tokens: i64,
     capabilities: &MessagesModelCapabilities,
 ) -> EffortLevel {
-    if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh {
+    let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier);
+    if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh {
         return EffortLevel::Xhigh;
     }
-    if budget_tokens >= budgets.high {
+    if reaches(budgets.high) {
         return EffortLevel::High;
     }
-    if budget_tokens >= budgets.medium {
+    if reaches(budgets.medium) {
         return EffortLevel::Medium;
     }
     EffortLevel::Low
@@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) -
     }
 }
 
+/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string
+/// as a budget, and nothing else.
+fn numeric_budget_tokens(budget: &Recognized<u64>) -> Option<i64> {
+    let value = match budget {
+        Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)),
+        Recognized::Unrecognized(value) => value,
+    };
+    match value {
+        Value::Number(number) => number
+            .as_i64()
+            .or_else(|| number.as_f64().map(|float| float.trunc() as i64)),
+        Value::String(text) => python_int(text),
+        Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None,
+    }
+}
+
+/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits.
+fn python_int(text: &str) -> Option<i64> {
+    let trimmed = text.trim();
+    let digits = trimmed.trim_start_matches(['+', '-']);
+    let well_formed = trimmed.len() - digits.len() <= 1
+        && !digits.is_empty()
+        && !digits.starts_with('_')
+        && !digits.ends_with('_')
+        && !digits.contains("__")
+        && digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_');
+    if !well_formed {
+        return None;
+    }
+    let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect();
+    cleaned.parse::<i64>().ok().or_else(|| {
+        Some(if cleaned.starts_with('-') {
+            i64::MIN
+        } else {
+            i64::MAX
+        })
+    })
+}
+
+fn python_type_name(value: &Value) -> &'static str {
+    match value {
+        Value::Null => "NoneType",
+        Value::Bool(_) => "bool",
+        Value::Number(number) if number.is_f64() => "float",
+        Value::Number(_) => "int",
+        Value::String(_) => "str",
+        Value::Array(_) => "list",
+        Value::Object(_) => "dict",
+    }
+}
+
+fn kept_display(
+    display: Option<&Recognized<ThinkingDisplay>>,
+    context: &ThinkingContext,
+) -> Option<Recognized<ThinkingDisplay>> {
+    match display? {
+        Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => {
+            display.cloned()
+        }
+        Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => {
+            display.cloned()
+        }
+        Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None,
+    }
+}
+
+/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive
+/// keys always, and any other key unless `drop_params` is set.
+fn carried_thinking_keys(
+    extra: &serde_json::Map<String, Value>,
+    context: &ThinkingContext,
+) -> serde_json::Map<String, Value> {
+    extra
+        .iter()
+        .filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params)
+        .map(|(key, value)| (key.clone(), value.clone()))
+        .collect()
+}
+
+/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`.
 fn translate_legacy_thinking_for_adaptive_model(
     request: MessagesRequest,
     context: &ThinkingContext,
-) -> MessagesRequest {
+) -> Result<MessagesRequest, Error> {
     let capabilities = &context.capabilities;
     if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking {
-        return request;
+        return Ok(request);
     }
     let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else {
-        return request;
+        return Ok(request);
     };
-    let budget = enabled
-        .budget_tokens
-        .as_ref()
-        .and_then(Recognized::known)
-        .copied()
-        .unwrap_or(0);
-    let level = effort_for_budget(&context.budgets, budget, capabilities);
-    MessagesRequest {
+    let budget = match &enabled.budget_tokens {
+        None => Some(0),
+        Some(budget) => numeric_budget_tokens(budget),
+    };
+    if budget.is_none() && !context.drop_params {
+        let actual = match &enabled.budget_tokens {
+            Some(Recognized::Unrecognized(value)) => python_type_name(value),
+            Some(Recognized::Known(_)) | None => "int",
+        };
+        return Err(Error::InvalidRequest(
+            crate::ErrorDetail::NonNumericThinkingBudget { actual },
+        ));
+    }
+    let thinking = ThinkingConfig::Adaptive(AdaptiveThinking {
+        display: kept_display(enabled.display.as_ref(), context),
+        extra: carried_thinking_keys(&enabled.extra, context),
+    });
+    let output_config = match budget {
+        Some(budget) => with_default_effort(
+            request.params.output_config,
+            effort_for_budget(&context.budgets, budget, capabilities),
+        ),
+        None => request.params.output_config,
+    };
+    Ok(MessagesRequest {
         params: MessagesOptionalParams {
-            thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))),
-            output_config: with_default_effort(request.params.output_config, level),
+            thinking: Some(Recognized::Known(thinking)),
+            output_config,
             ..request.params
         },
         ..request
-    }
+    })
 }
 
 fn translate_adaptive_effort_for_non_adaptive_model(
@@ -337,7 +435,7 @@ pub fn translate_thinking(
 ) -> Result<MessagesRequest, Error> {
     let request = translate_reasoning_effort(request, context)?;
     let request = drop_disabled_thinking(request, context);
-    let request = translate_legacy_thinking_for_adaptive_model(request, context);
+    let request = translate_legacy_thinking_for_adaptive_model(request, context)?;
     let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?;
     Ok(drop_incompatible_temperature_for_thinking(request, context))
 }
@@ -363,6 +461,7 @@ mod tests {
         ThinkingContext {
             capabilities,
             budgets: ThinkingBudgets::default(),
+            ..ThinkingContext::default()
         }
     }
 
@@ -1239,10 +1338,102 @@ mod tests {
         let context = ThinkingContext {
             capabilities,
             budgets: overridden_budgets(overrides),
+            ..ThinkingContext::default()
         };
         assert_eq!(
             translate_thinking(request(input), &context),
             Ok(request(expected))
         );
     }
+
+    use serde_json::json;
+
+    fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext {
+        ThinkingContext {
+            drop_params,
+            keeps_display_updates,
+            ..context(opus_4_7())
+        }
+    }
+
+    #[rstest]
+    #[case::float_budget(json!(8192.0), "xhigh")]
+    #[case::float_below_a_tier(json!(1023.9), "low")]
+    #[case::string_budget(json!(" 16_000 "), "xhigh")]
+    #[case::negative_budget(json!(-5), "low")]
+    fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) {
+        let translated = translate_thinking(
+            request(
+                json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}),
+            ),
+            &legacy_context(false, false),
+        );
+        assert_eq!(
+            translated,
+            Ok(request(json!({
+                "max_tokens": 64000,
+                "thinking": {"type": "adaptive"},
+                "output_config": {"effort": effort},
+            })))
+        );
+    }
+
+    #[rstest]
+    #[case::text(json!("lots"), "str")]
+    #[case::list(json!([1]), "list")]
+    #[case::null(Value::Null, "NoneType")]
+    #[case::boolean(json!(true), "bool")]
+    fn a_non_numeric_legacy_budget_is_rejected_unless_dropped(
+        #[case] budget: Value,
+        #[case] actual: &'static str,
+    ) {
+        let fields =
+            json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}});
+        assert_eq!(
+            translate_thinking(request(fields.clone()), &legacy_context(false, false)),
+            Err(Error::InvalidRequest(
+                crate::ErrorDetail::NonNumericThinkingBudget { actual }
+            ))
+        );
+        assert_eq!(
+            translate_thinking(request(fields), &legacy_context(true, false)),
+            Ok(request(
+                json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}})
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::summarized("summarized", false, Some("summarized"))]
+    #[case::updates_kept("updates", true, Some("updates"))]
+    #[case::updates_dropped("updates", false, None)]
+    #[case::unknown("verbose", true, None)]
+    fn legacy_display_survives_only_where_python_keeps_it(
+        #[case] display: &str,
+        #[case] keeps_display_updates: bool,
+        #[case] kept: Option<&str>,
+    ) {
+        let translated = translate_thinking(
+            request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})),
+            &legacy_context(false, keeps_display_updates),
+        )
+        .unwrap();
+        let thinking = serde_json::to_value(translated.params.thinking).unwrap();
+        assert_eq!(thinking.get("display").and_then(Value::as_str), kept);
+    }
+
+    #[rstest]
+    #[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))]
+    #[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))]
+    fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) {
+        let translated = translate_thinking(
+            request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})),
+            &legacy_context(drop_params, false),
+        )
+        .unwrap();
+        assert_eq!(
+            serde_json::to_value(translated.params.thinking).unwrap(),
+            expected
+        );
+    }
 }

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         request: MessagesRequest,
         context: &MessagesTransformContext,
     ) -> Result<MessagesRequest, Error> {
-        transform_messages_request(request, context)
+        transform_messages_request(request, &context.keeping_display_updates())
     }
 
     fn secret_names(&self) -> &'static [&'static str] {
@@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
             .then_some(AnthropicBeta::PerTurnControl20260701),
         uses_thinking_display_updates(params)
             .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
+        uses_thinking_block_binding(params)
+            .then_some(AnthropicBeta::ThinkingBindingControls20260801),
         messages_carry_tool_changes(&request.messages)
             .then_some(AnthropicBeta::MidConversationToolChanges20260701),
         has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
@@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
     display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
 }
 
+/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`.
+fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool {
+    match params.thinking.as_ref() {
+        Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Unrecognized(Value::Object(thinking))) => {
+            thinking.contains_key("block_binding")
+        }
+        Some(Recognized::Unrecognized(_)) | None => false,
+    }
+}
+
 fn is_tool_reference_change(block: &ContentBlock) -> bool {
     matches!(
         &block.block_type,
@@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error
     })
 }
 
-fn drop_unsupported_params(
+/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`.
+pub fn drop_unsupported_params(
     request: MessagesRequest,
     context: &MessagesTransformContext,
 ) -> Result<MessagesRequest, Error> {
@@ -838,6 +860,8 @@ mod tests {
                             low: 2000,
                             ..ThinkingBudgets::default()
                         },
+                        drop_params: true,
+                        keeps_display_updates: false,
                     },
                     drop_params: true,
                 }

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
                     .capabilities
                     .supports_mid_conversation_system,
             )),
-            context,
+            &context.keeping_display_updates(),
         )
     }
 

litellm-rust/crates/llms/src/base_llm/messages/context.rs

@@ -110,6 +110,11 @@ impl ThinkingBudgets {
 pub struct ThinkingContext {
     pub capabilities: MessagesModelCapabilities,
     pub budgets: ThinkingBudgets,
+    /// `drop_params` as Python's thinking translation reads it.
+    pub drop_params: bool,
+    /// The provider accepts `display: "updates"` on thinking translated from the legacy shape
+    /// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`).
+    pub keeps_display_updates: bool,
 }
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub struct MessagesTransformContext {
@@ -130,8 +135,22 @@ impl MessagesTransformContext {
             thinking: ThinkingContext {
                 capabilities,
                 budgets: ThinkingBudgets::from_lookup(env),
+                drop_params,
+                keeps_display_updates: false,
             },
             drop_params,
         }
     }
+
+    /// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`.
+    #[must_use]
+    pub fn keeping_display_updates(&self) -> Self {
+        Self {
+            thinking: ThinkingContext {
+                keeps_display_updates: true,
+                ..self.thinking
+            },
+            ..*self
+        }
+    }
 }

litellm-rust/crates/llms/src/error.rs

@@ -92,6 +92,24 @@ impl Error {
         }
     }
 
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match self {
+            Self::Http(error) => error.upstream_headers(),
+            _ => &[],
+        }
+    }
+
+    pub fn is_body_read_timeout(&self) -> bool {
+        matches!(self, Self::Http(error) if error.is_body_read_timeout())
+    }
+
+    pub fn is_timeout(&self) -> Option<bool> {
+        match self {
+            Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+            _ => None,
+        }
+    }
+
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match self {
             Self::Http(error) => error.upstream_response(),
@@ -158,6 +176,10 @@ pub enum ErrorDetail {
         value: String,
         hint: String,
     },
+    #[error(
+        "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy"
+    )]
+    NonNumericThinkingBudget { actual: &'static str },
     #[error(
         "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params"
     )]

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         request: MessagesRequest,
         context: &MessagesTransformContext,
     ) -> Result<MessagesRequest, Error> {
-        let request = transform_messages_request(request, context)?;
+        let request = transform_messages_request(request, &context.keeping_display_updates())?;
         let request = strip_cache_control_scope(normalize_system_role_messages(
             request,
             context

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -186,6 +186,9 @@ pub struct ModelInfoFields {
     pub supports_image_size: Option<bool>,
     pub supported_audio_formats: Option<Value>,
     pub token_count_framing: Option<i64>,
+    pub token_count_tool_framing: Option<i64>,
+    pub token_count_responses_framing: Option<i64>,
+    pub token_count_responses_tool_framing: Option<i64>,
     pub token_count_model: Option<Value>,
     pub token_count_tokenizer: Option<Value>,
     pub vertex_ai_audio_api: Option<Value>,

litellm-rust/crates/model-catalog/src/vocabulary.rs

@@ -93,6 +93,12 @@ pub enum Endpoint {
     Realtime,
     #[serde(rename = "/v1/realtime/transcription_sessions")]
     RealtimeTranscriptionSessions,
+    #[serde(rename = "/v1/realtime/translations")]
+    RealtimeTranslations,
+    #[serde(rename = "/v1/realtime/translations/calls")]
+    RealtimeTranslationCalls,
+    #[serde(rename = "/v1/realtime/translations/client_secrets")]
+    RealtimeTranslationClientSecrets,
     #[serde(rename = "/v1/videos")]
     Videos,
     #[serde(rename = "/v1/ocr")]

litellm-rust/crates/model-catalog/src/wire/attributes.rs

@@ -11,7 +11,7 @@ use crate::error::{DateCause, Error, Repr};
 use crate::vocabulary::{
     AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
 };
-use crate::wire::lax::Count;
+use crate::wire::lax::{Count, NumericCount};
 
 macro_rules! attributes {
     ($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
@@ -104,9 +104,14 @@ attributes! {
     /// Cloud regions the model is available in ('global' or region ids).
     supported_regions: Vec<String>,
     thinking_always_on: bool,
-    token_count_framing: u64 as Count,
+    /// Whether the model thinks when the request carries no thinking param.
+    thinking_on_by_default: bool,
+    token_count_framing: u64 as NumericCount,
     token_count_model: TokenCountModel,
+    token_count_responses_framing: u64 as NumericCount,
+    token_count_responses_tool_framing: u64 as NumericCount,
     token_count_tokenizer: TokenCountTokenizer,
+    token_count_tool_framing: u64 as NumericCount,
     /// Provider default tokens-per-minute limit.
     tpm: u64 as Count,
     use_openai_responses_path: bool,
@@ -298,10 +303,24 @@ mod tests {
     #[case::known_mode("mode", json!("chat"), true)]
     #[case::unknown_mode("mode", json!("chatt"), false)]
     #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
+    #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
+    #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
     #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
     #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
     #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
     #[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
+    #[case::framing("token_count_framing", json!(12), true)]
+    #[case::framing_written_as_text("token_count_framing", json!("12"), false)]
+    #[case::framing_written_as_whole_float("token_count_framing", json!(12.0), true)]
+    #[case::fractional_tool_framing("token_count_tool_framing", json!(1.5), false)]
+    #[case::tool_framing("token_count_tool_framing", json!(214), true)]
+    #[case::tool_framing_written_as_text("token_count_tool_framing", json!("214"), false)]
+    #[case::tool_framing_as_a_flag("token_count_tool_framing", json!(true), false)]
+    #[case::negative_tool_framing("token_count_tool_framing", json!(-1), false)]
+    #[case::responses_framing("token_count_responses_framing", json!(38), true)]
+    #[case::responses_framing_as_a_flag("token_count_responses_framing", json!(false), false)]
+    #[case::responses_tool_framing("token_count_responses_tool_framing", json!(267), true)]
+    #[case::responses_tool_framing_written_as_text("token_count_responses_tool_framing", json!("267"), false)]
     fn each_attribute_reads_its_own_type(
         #[case] key: &str,
         #[case] value: Value,

litellm-rust/crates/model-catalog/src/wire/lax.rs

@@ -8,6 +8,8 @@ pub(crate) struct LaxI64;
 
 pub(crate) struct Count;
 
+pub(crate) struct NumericCount;
+
 impl<'de> DeserializeAs<'de, i64> for LaxI64 {
     fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
         deserializer.deserialize_any(Self)
@@ -52,6 +54,34 @@ impl<'de> DeserializeAs<'de, u64> for Count {
     }
 }
 
+impl<'de> DeserializeAs<'de, u64> for NumericCount {
+    fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
+        deserializer.deserialize_any(Self)
+    }
+}
+
+impl Visitor<'_> for NumericCount {
+    type Value = u64;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("a whole number of zero or more")
+    }
+
+    fn visit_u64<E: de::Error>(self, value: u64) -> Result<u64, E> {
+        Ok(value)
+    }
+
+    fn visit_i64<E: de::Error>(self, value: i64) -> Result<u64, E> {
+        u64::try_from(value).map_err(|_| E::invalid_value(de::Unexpected::Signed(value), &self))
+    }
+
+    fn visit_f64<E: de::Error>(self, value: f64) -> Result<u64, E> {
+        integral_float(value)
+            .and_then(|whole| u64::try_from(whole).ok())
+            .ok_or_else(|| E::invalid_value(de::Unexpected::Float(value), &self))
+    }
+}
+
 fn integer_string(value: &str) -> Option<i64> {
     let integer = match value.split_once('.') {
         Some((integer, fraction)) => {
@@ -99,6 +129,12 @@ mod tests {
             .map(DeserializeAsWrap::into_inner)
     }
 
+    fn numeric_count(value: Value) -> Option<u64> {
+        DeserializeAsWrap::<u64, NumericCount>::deserialize(value)
+            .ok()
+            .map(DeserializeAsWrap::into_inner)
+    }
+
     use serde::Deserialize;
 
     #[rstest]
@@ -125,4 +161,23 @@ mod tests {
     ) {
         assert_eq!(count(written), expected);
     }
+
+    #[rstest]
+    #[case::integer(json!(214), Some(214))]
+    #[case::zero(json!(0), Some(0))]
+    #[case::whole_float(json!(214.0), Some(214))]
+    #[case::negative_zero(json!(-0.0), Some(0))]
+    #[case::above_i64(json!(u64::MAX), Some(u64::MAX))]
+    #[case::negative(json!(-1), None)]
+    #[case::negative_whole_float(json!(-1.0), None)]
+    #[case::fraction(json!(1.5), None)]
+    #[case::string(json!("214"), None)]
+    #[case::boolean(json!(true), None)]
+    #[case::list(json!([214]), None)]
+    fn numeric_counts_accept_only_whole_json_numbers(
+        #[case] written: Value,
+        #[case] expected: Option<u64>,
+    ) {
+        assert_eq!(numeric_count(written), expected);
+    }
 }

litellm-rust/crates/python-bridge/src/errors.rs

@@ -23,6 +23,17 @@ pyo3::create_exception!(
 );
 
 const MESSAGES_REQUEST_ERROR: &str = "messages_request_error";
+/// The rejection is a parameter the model does not support, which Python raises as
+/// `UnsupportedParamsError`.
+const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param";
+/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or
+/// `Body` for a streamed body read that timed out after the head arrived.
+const MESSAGES_TIMEOUT: &str = "messages_timeout";
+/// No credential reached the provider, which Python raises as `AuthenticationError`.
+const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential";
+/// Python refuses the request only once its logger knows the resolved model and provider (the
+/// provider config's request transform), so the failure logs them.
+const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection";
 
 pub(crate) fn upstream_error(
     py: Python<'_>,
@@ -78,17 +89,57 @@ pub(crate) fn route_failure(
         return Ok(raised);
     }
     if let Some((status, body)) = error.upstream_response() {
-        return public(upstream_error(py, status, body, &HeaderMap::new())?);
+        let headers = error
+            .upstream_headers()
+            .iter()
+            .filter_map(|(name, value)| {
+                Some((
+                    reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?,
+                    reqwest::header::HeaderValue::from_str(value).ok()?,
+                ))
+            })
+            .collect::<HeaderMap>();
+        return public(upstream_error(py, status, body, &headers)?);
     }
     let native = match (marker, error.rejected_request()) {
         (RequestMarker::Messages, Some(rejection)) => {
             let native = PyValueError::new_err(Report::new(rejection).to_string());
-            native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?;
+            let value = native.value(py);
+            value.setattr(MESSAGES_REQUEST_ERROR, true)?;
+            if matches!(
+                rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
+                Some(
+                    litellm_llms::ErrorDetail::UnsupportedParameter { .. }
+                        | litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }
+                )
+            ) {
+                value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?;
+            }
+            if matches!(
+                rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
+                Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. })
+            ) {
+                value.setattr(MESSAGES_LATE_REJECTION, true)?;
+            }
             native
         }
-        (RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => {
-            by_fault(error.is_request(), &error)
+        (RequestMarker::Messages, None) => {
+            let native = by_fault(error.is_request(), &error);
+            let value = native.value(py);
+            if let Some(connect) = error.timeout() {
+                let phase = match (connect, error.body_read_timeout()) {
+                    (true, _) => "Connection",
+                    (false, true) => "Body",
+                    (false, false) => "Read",
+                };
+                value.setattr(MESSAGES_TIMEOUT, phase)?;
+            }
+            if error.is_missing_credential() {
+                value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?;
+            }
+            native
         }
+        (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error),
     };
     public(native)
 }

litellm-rust/crates/python-bridge/src/routes/messages/host.rs

@@ -34,11 +34,9 @@ fn merge_headers(
     forwarded: Option<Map<String, Value>>,
     extra_headers: Option<Map<String, Value>>,
 ) -> Option<Map<String, Value>> {
-    let merged: Map<String, Value> = forwarded
-        .into_iter()
-        .flatten()
-        .chain(extra_headers.into_iter().flatten())
-        .collect();
+    let merged = litellm_inference_messages::merge_messages_header_sources(
+        forwarded.into_iter().chain(extra_headers),
+    );
     (!merged.is_empty()).then_some(merged)
 }
 
@@ -107,10 +105,19 @@ impl MessagesPythonHost {
             .map(|value| python_timeout_seconds(py, value.unbind()))
             .transpose()?
             .flatten();
-        let custom_llm_provider = string("custom_llm_provider")?;
-        let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
         let api_key = string("api_key")?;
         let api_base = string("api_base")?;
+        // Python's handler resolves a bare model name (`claude-sonnet-4-5`) through
+        // `get_llm_provider`'s model lists, which the native prefix lookup does not carry.
+        let custom_llm_provider = match string("custom_llm_provider")? {
+            Some(provider) => Some(provider),
+            None => py
+                .import(ROUTE_HOST_MODULE)?
+                .getattr("inferred_provider")?
+                .call1((&model, api_base.as_deref()))?
+                .extract()?,
+        };
+        let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
         let extra_headers = self.merged_headers(py, arguments)?;
         let provider_specific_header = self.provider_specific_header(py, arguments)?;
         let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;

litellm-rust/crates/python-bridge/src/routes/messages/mod.rs

@@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
     )
 }
 
+/// Python's sync Messages handler raises for every native provider before it sends anything
+/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies
+/// that until both sides serve sync calls.
+const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls";
+
 #[pyfunction]
-pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
-    run_messages(py, call, false)
+#[pyo3(signature = (call))]
+pub(crate) fn messages(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
+    drop(call);
+    Err(pyo3::exceptions::PyValueError::new_err(
+        SYNC_NOT_IMPLEMENTED,
+    ))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -8,7 +8,7 @@ _apply_process_defaults()
 import threading
 import os
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from typing import (
     Any,
     Callable,
@@ -1562,7 +1562,9 @@ from .realtime_api.main import (
     _arealtime,
     acreate_realtime_client_secret,
     acreate_realtime_transcription_session,
+    acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
     arealtime_calls,
+    arealtime_translation_calls as arealtime_translation_calls,
 )
 from .responses.main import _aresponses_websocket
 
@@ -1647,6 +1649,7 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
 # Lazy loading system for heavy modules to reduce initial import time and memory usage
 
 if TYPE_CHECKING:
+    from . import guardrails as guardrails
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -1753,6 +1756,9 @@ if TYPE_CHECKING:
     from .llms.typesafe.decisions.transformation import (
         TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
     )
+    from .llms.parallel_ai.decisions.transformation import (
+        ParallelAIDecisionsConfig as ParallelAIDecisionsConfig,
+    )
     from .llms.openrouter.decisions.transformation import (
         OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
     )
@@ -1807,6 +1813,9 @@ if TYPE_CHECKING:
     from .llms.vertex_ai.rerank.transformation import (
         VertexAIRerankConfig as VertexAIRerankConfig,
     )
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.fireworks_ai.rerank.transformation import (
         FireworksAIRerankConfig as FireworksAIRerankConfig,
     )
@@ -1852,9 +1861,6 @@ if TYPE_CHECKING:
         BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
-    from .llms.together_ai.chat.transformation import (
-        TogetherAIChatConfig as TogetherAIChatConfig,
-    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -2013,6 +2019,9 @@ if TYPE_CHECKING:
     from .llms.xai.responses.transformation import (
         XAIResponsesAPIConfig as XAIResponsesAPIConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.litellm_proxy.responses.transformation import (
         LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
     )
@@ -2040,9 +2049,6 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
-    from .llms.vertex_ai.interactions.transformation import (
-        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
-    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,
@@ -2373,6 +2379,7 @@ if TYPE_CHECKING:
     supports_web_search: Callable[..., bool]
     supports_tool_search: Callable[..., bool]
     supports_url_context: Callable[..., bool]
+    supports_continuation: Callable[..., bool]
     supports_response_schema: Callable[..., bool]
     supports_parallel_function_calling: Callable[..., bool]
     supports_vision: Callable[..., bool]
@@ -2499,10 +2506,10 @@ def __getattr__(name: str) -> Any:
         handler_func: Final = registry[name]
         return handler_func(name)
 
-    if name == "proxy":
+    if name in ("proxy", "guardrails"):
         import importlib
 
-        return importlib.import_module("litellm.proxy")
+        return importlib.import_module(f"litellm.{name}")
 
     if name == "harness" or name in _AGENT_EXPORTS:
         import importlib

litellm/_internal_context.py

@@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
         internal_parent_request_id.reset(parent_token)
 
 
+@contextmanager
+def billed_separately() -> Generator[None]:
+    """A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
+    token: Final = is_internal_call.set(False)
+    parent_token: Final = internal_parent_request_id.set(None)
+    try:
+        yield
+    finally:
+        is_internal_call.reset(token)
+        internal_parent_request_id.reset(parent_token)
+
+
 @contextmanager
 def post_response_phase() -> Generator[None]:
     """Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

litellm/_lazy_imports_registry.py

@@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
     "supports_web_search",
     "supports_tool_search",
     "supports_url_context",
+    "supports_continuation",
     "supports_response_schema",
     "supports_parallel_function_calling",
     "supports_vision",
@@ -162,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
     "HostedVLLMRerankConfig",
     "PerplexityDecisionsConfig",
     "TypeSafeDecisionsConfig",
+    "ParallelAIDecisionsConfig",
     "NaceDecisionsConfig",
     "OpenRouterDecisionsConfig",
     "FireworksAIDecisionsConfig",
@@ -488,6 +490,7 @@ _UTILS_IMPORT_MAP: Final = {
     "supports_web_search": (".utils", "supports_web_search"),
     "supports_tool_search": (".utils", "supports_tool_search"),
     "supports_url_context": (".utils", "supports_url_context"),
+    "supports_continuation": (".utils", "supports_continuation"),
     "supports_response_schema": (".utils", "supports_response_schema"),
     "supports_parallel_function_calling": (
         ".utils",
@@ -738,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
     "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
     "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
+    "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
     "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
     "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
     "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),

litellm/anthropic_beta_headers_config.json

@@ -37,7 +37,47 @@
     "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "azure_ai": {
     "advisor-tool-2026-03-01": null,
@@ -45,23 +85,21 @@
     "bash_20241022": null,
     "bash_20250124": null,
     "code-execution-2025-08-25": "code-execution-2025-08-25",
-    "compact-2026-01-12": null,
-    "compact-2026-09-04": "compact-2026-09-04",
+    "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": null,
+    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": "mcp-client-2025-11-20",
     "mcp-client-2025-04-04": "mcp-client-2025-04-04",
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@@ -71,9 +109,54 @@
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": "web-fetch-2025-09-10",
-    "web-search-2025-03-05": "web-search-2025-03-05"
+    "web-search-2025-03-05": "web-search-2025-03-05",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "compact-2026-09-04": "compact-2026-09-04",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "bedrock_converse": {
     "advisor-tool-2026-03-01": null,
@@ -196,34 +279,92 @@
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": null,
     "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
-    "structured-outputs-2025-11-13": null,
+    "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": null,
+    "cache-diagnosis-2026-04-07": null,
+    "ce-plugins-2026-09-01": null,
+    "ce-user-management-2026-07-13": null,
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": null,
+    "compact-2026-09-04": null,
+    "computer-use-2024-10-22": null,
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": null,
+    "experimental-claude-tag-2026-10-21": null,
+    "extended-cache-ttl-2025-04-11": null,
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": null,
+    "mcp-client-2026-09-15": null,
+    "mcp-tunnels-2026-06-22": null,
+    "message-batches-2024-09-24": null,
+    "message-threads-2026-08-12": null,
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": null,
+    "output-300k-2026-03-24": null,
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": null,
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": null,
+    "spend-limit-reads-2026-09-26": null,
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": null,
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "user-profiles-2026-03-24": null,
+    "user-profiles-2026-08-18": null,
+    "user-profiles-2026-09-04": null,
+    "webhooks-2026-10-07": null,
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": null,
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "vertex_ai_header_only": {
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
     "compact-2026-09-04": "compact-2026-09-04",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04"
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,

litellm/anthropic_beta_headers_manager.py

@@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import cache
 from importlib.resources import files
 from typing import Final, Literal, TypeAlias
+from urllib.parse import urlparse
 
 import httpx
 from pydantic import TypeAdapter
@@ -48,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 _DECODED_JSON: Final = TypeAdapter(object)
 
 BetaPlacement: TypeAlias = Literal["header", "body"]
+UnlistedBetas: TypeAlias = Literal["drop", "forward"]
 _HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
 
 
@@ -280,12 +282,13 @@ def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
     placement: BetaPlacement = "header",
+    unlisted: UnlistedBetas = "drop",
 ) -> list[str]:
     """
     Filter and transform beta headers based on provider's mapping configuration.
 
     This function:
-    1. Only allows headers that are present in the provider's mapping keys
+    1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
     2. Filters out headers with null values (unsupported)
     3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
 
@@ -293,6 +296,7 @@ def filter_and_transform_beta_headers(
         beta_headers: List of Anthropic beta header values
         provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
         placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
+        unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
 
     Returns:
         List of filtered and transformed beta headers for the provider
@@ -307,6 +311,10 @@ def filter_and_transform_beta_headers(
     for header in beta_headers:
         header = header.strip()
 
+        if header not in provider_mapping and unlisted == "forward":
+            filtered_headers.add(header)
+            continue
+
         # Check if header is in the mapping
         if header not in provider_mapping:
             verbose_logger.debug(
@@ -388,9 +396,38 @@ def get_provider_beta_header(
     return provider_mapping[anthropic_beta_header]
 
 
+FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
+    "services.ai.azure.com",
+    "openai.azure.com",
+    "cognitiveservices.azure.com",
+)
+
+
+def is_foundry_api_base(api_base: str | None) -> bool:
+    """Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft."""
+    if not api_base:
+        return False
+    host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or ""
+    return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES)
+
+
+def resolve_beta_provider(provider: str, api_base: str | None = None) -> str:
+    """
+    Pick the beta allowlist row for one deployment, not one provider string.
+
+    An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and
+    gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com.
+    """
+    if provider == "anthropic" and is_foundry_api_base(api_base):
+        return "azure_ai"
+    return provider
+
+
 def update_headers_with_filtered_beta(
     headers: dict,
     provider: str,
+    api_base: str | None = None,
+    unlisted: UnlistedBetas = "drop",
 ) -> dict:
     """
     Update headers dict by filtering and transforming anthropic-beta header values.
@@ -399,6 +436,8 @@ def update_headers_with_filtered_beta(
     Args:
         headers: Request headers dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
+        unlisted: Whether a beta missing from the row is dropped or forwarded as sent
 
     Returns:
         Updated headers dict
@@ -413,7 +452,8 @@ def update_headers_with_filtered_beta(
     # Filter and transform based on provider
     filtered_beta_values: Final = filter_and_transform_beta_headers(
         beta_headers=beta_values,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
+        unlisted=unlisted,
     )
 
     # Update or remove the header
@@ -430,6 +470,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
+    api_base: str | None = None,
 ) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
@@ -439,11 +480,12 @@ def update_request_with_filtered_beta(
         headers: Request headers dict (will be modified in place)
         request_data: Request body dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
 
     Returns:
         Tuple of (updated headers, updated request_data)
     """
-    headers = update_headers_with_filtered_beta(headers=headers, provider=provider)
+    headers = update_headers_with_filtered_beta(headers=headers, provider=provider, api_base=api_base)
 
     existing_body_betas: Final = request_data.get("anthropic_beta")
     if not existing_body_betas:
@@ -451,7 +493,7 @@ def update_request_with_filtered_beta(
 
     filtered_body_betas: Final = filter_and_transform_beta_headers(
         beta_headers=existing_body_betas,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
         placement="body",
     )
 

litellm/anthropic_interface/exceptions/exception_mapping_utils.py

@@ -16,7 +16,7 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE
 _PARSED_ERROR_BODY: Final = TypeAdapter(object)
 
 # HTTP status code -> Anthropic error type
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
     400: "invalid_request_error",
     401: "authentication_error",
@@ -46,6 +46,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Create an Anthropic-formatted error response dict.
@@ -57,12 +58,10 @@ class AnthropicExceptionMapping:
             "request_id": "req_..."
         }
         """
-        error_type: Final = AnthropicExceptionMapping.get_error_type(status_code)
-
         response: Final[AnthropicErrorResponse] = {
             "type": "error",
             "error": {
-                "type": error_type,
+                "type": error_type or AnthropicExceptionMapping.get_error_type(status_code),
                 "message": message,
             },
         }
@@ -131,6 +130,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         raw_message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Transform an error message to Anthropic format.
@@ -144,6 +144,7 @@ class AnthropicExceptionMapping:
             status_code: HTTP status code
             raw_message: Raw error message (may be JSON string or plain text)
             request_id: Optional request ID to include
+            error_type: Anthropic error type to use instead of the one the status maps to
 
         Returns:
             AnthropicErrorResponse dict
@@ -169,6 +170,7 @@ class AnthropicExceptionMapping:
             status_code=status_code,
             message=message,
             request_id=request_id,
+            error_type=error_type,
         )
 
 

litellm/anthropic_interface/exceptions/exceptions.py

@@ -6,15 +6,18 @@ from typing import Literal
 from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 # Known Anthropic error types
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 AnthropicErrorType = Literal[
     "invalid_request_error",
     "authentication_error",
+    "billing_error",
     "permission_error",
     "not_found_error",
+    "conflict_error",
     "request_too_large",
     "rate_limit_error",
     "api_error",
+    "timeout_error",
     "overloaded_error",
 ]
 
@@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict):
     type: AnthropicErrorType
     message: str
     provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
+    litellm_error_type: NotRequired[ReadOnly[str]]
     litellm_call_id: NotRequired[ReadOnly[str]]
 
 

litellm/caching/caching.py

@@ -45,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
 from .s3_cache import S3Cache
 
 RESPONSE_CACHE_TARGET: Final = "llm_response"
-_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
+_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
 _DECODED_LITERAL: Final = TypeAdapter(object)
 _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
     list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
 )
 
 
+def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
+    raw_extra_body: Final = kwargs.get("extra_body")
+    if not isinstance(raw_extra_body, dict):
+        return ""
+    extra_body: Final = cast(  # cast-ok: isinstance leaves the key and value types unknown
+        Mapping[str, object], raw_extra_body
+    )
+    return "".join(
+        f"extra_body.{param}: {extra_body[param]}"
+        for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
+        if extra_body.get(param)
+    )
+
+
 @contextmanager
 def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
     """The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
@@ -429,7 +443,10 @@ class Cache:
             "" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
         )
         hashed_cache_key = Cache._get_hashed_cache_key(
-            cache_key + reasoning_summary_material + tool_call_id_shape_material
+            cache_key
+            + _keyed_provider_params_in_extra_body(kwargs)
+            + reasoning_summary_material
+            + tool_call_id_shape_material
         )
         hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
         verbose_logger.debug(

litellm/caching/dual_cache.py

@@ -673,7 +673,7 @@ class DualCache(BaseCache):
         if self.redis_cache is not None:
             self.redis_cache.flush_cache()
 
-    def delete_cache(self, key):
+    def delete_cache(self, key: str) -> None:
         """
         Delete a key from the cache
         """

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -11,7 +11,7 @@ from typing import (
     cast,  # noqa: TID251  # optional_params reaches the bridge as an untyped dict
 )
 
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.types.llms.openai import ResponsesAPIResponse
 
@@ -20,15 +20,15 @@ if TYPE_CHECKING:
 
 
 class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
-    model: str
-    messages: list
-    optional_params: dict
-    litellm_params: dict
-    headers: dict
-    model_response: "ModelResponse"
-    logging_obj: "LiteLLMLoggingObj"
-    custom_llm_provider: str
-    encoding: object
+    model: ReadOnly[str]
+    messages: ReadOnly[list]
+    optional_params: ReadOnly[dict[str, object]]
+    litellm_params: ReadOnly[dict[str, object]]
+    headers: ReadOnly[dict]
+    model_response: ReadOnly["ModelResponse"]
+    logging_obj: ReadOnly["LiteLLMLoggingObj"]
+    custom_llm_provider: ReadOnly[str]
+    encoding: ReadOnly[object]
 
 
 def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler:
         custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
         if kwargs.get("stream") is True and "stream" not in optional_params:
             optional_params = {**optional_params, "stream": True}
+        logging_obj.record_bridged_chat_request(optional_params, litellm_params)
 
         request_data: Final = self.transformation_handler.transform_request(
             model=model,
@@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler:
         custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
         if kwargs.get("stream") is True and "stream" not in optional_params:
             optional_params = {**optional_params, "stream": True}
+        logging_obj.record_bridged_chat_request(optional_params, litellm_params)
 
         try:
             request_data: Final = self.transformation_handler.transform_request(

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import (
     ResponseCustomToolCallOutputParam,
     ResponseCustomToolCallParam,
 )
+from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
 from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
 from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
 from openai.types.responses.tool_param import FunctionToolParam
@@ -33,11 +34,17 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
     replay_items,
     requested_phase,
 )
-from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
+from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
+from litellm.litellm_core_utils.allowed_tools_tool_choice import (
+    parse_chat_allowed_tools_tool_choice,
+    responses_allowed_tools_tool_choice,
+)
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    prompt_cache_breakpoint_for_wire,
+    prompt_cache_breakpoint_with_readable_mode,
     responses_reasoning_items_from_thinking_blocks,
+    whole_prompt_cache_breakpoint_or_none,
     with_prompt_cache_breakpoint,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -97,6 +104,12 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
 )
 
 
+def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
+    if target_trims_breakpoints:
+        return prompt_cache_breakpoint_with_readable_mode(marker, drop_params)
+    return whole_prompt_cache_breakpoint_or_none(marker) if drop_params else marker
+
+
 def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
     if not isinstance(value, dict):
         return value
@@ -775,13 +788,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def _normalize_tool_choice_for_responses_api(
         self, tool_choice: _ToolChoiceT
-    ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
+    ) -> (
+        _ToolChoiceT
+        | ToolChoiceFunctionParam
+        | ToolChoiceCustomParam
+        | ToolChoiceAllowedParam
+        | Literal["auto", "none", "required"]
+    ):
         """Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
         if not isinstance(tool_choice, dict):
             return tool_choice
         choice_type: Final = tool_choice.get("type")
         if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
             return choice_type
+        if choice_type == "allowed_tools":
+            parsed: Final = parse_chat_allowed_tools_tool_choice(
+                cast(  # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
+                    object, tool_choice
+                )
+            )
+            flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None
+            if flat is not None:
+                return flat
         if choice_type not in ("function", "custom"):
             return tool_choice
         if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
@@ -844,6 +872,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
@@ -852,6 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
+            target_trims_breakpoints=target_trims_breakpoints,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
             replay_openai_history=replay_openai_history,
@@ -868,6 +898,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
         replay_openai_history: bool = False,
@@ -921,6 +952,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 content,
                                 role,
                                 drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                         }
                     )
@@ -940,6 +972,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         content,
                         "user",  # Use "user" role to get input_* types
                         drop_params=drop_params,
+                        target_trims_breakpoints=target_trims_breakpoints,
                     )
                 else:
                     # Fallback: convert unexpected types to input_text
@@ -968,7 +1001,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "type": "message",
                             "role": "assistant",
                             "content": self._convert_content_to_responses_format(
-                                content, "assistant", drop_params=drop_params
+                                content,
+                                "assistant",
+                                drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                             **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
@@ -1009,7 +1045,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(
-                            content, cast(str, role), drop_params=drop_params
+                            content,
+                            cast(str, role),
+                            drop_params=drop_params,
+                            target_trims_breakpoints=target_trims_breakpoints,
                         ),
                         **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
@@ -1127,14 +1166,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         optional_params = self._extract_extra_body_params(optional_params)
         provider: Final = _served_provider(litellm_params)
         base_model: Final = litellm_params.get("base_model")
-        supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or (
-            isinstance(base_model, str)
-            and bool(base_model)
-            and supports_openai_prompt_cache_breakpoint(base_model, provider)
+        model_info: Final = routed_model_info(litellm_params.get("model_info"))
+        named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,)
+        supports_prompt_cache_breakpoint: Final = any(
+            supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
         )
         converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
             messages,
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
+            target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
@@ -1195,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         api_model: Final = model
 
-        from litellm.types.utils import CallTypes
-
-        setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
-
         sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
 
         request_data: Final = {
@@ -1640,6 +1676,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         | None,
         role: str,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
     ) -> list[dict[str, object]]:
         """Convert chat completion content to responses API format"""
         from litellm.types.llms.openai import ChatCompletionImageObject
@@ -1666,7 +1703,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     if original_type == "text":
                         converted = with_prompt_cache_breakpoint(
                             self._convert_content_str_to_input_text(item.get("text", ""), role),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
@@ -1679,14 +1718,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                     cast(ChatCompletionImageObject, item), role
                                 ),
                             ),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
                     elif original_type == "input_audio":
                         converted = with_prompt_cache_breakpoint(
                             {"type": "input_audio", "input_audio": item.get("input_audio")},
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   input_audio -> %s", converted)
@@ -1702,7 +1745,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 _input_file_from_file_value(
                                     cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
                                 ),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1726,7 +1771,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             # Default to input_text for unknown types
                             converted = with_prompt_cache_breakpoint(
                                 self._convert_content_str_to_input_text(str(item.get("text", item)), role),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   unknown(%s) -> %s", original_type, converted)

litellm/constants.py

@@ -432,6 +432,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
 DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
+DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
 
 BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
@@ -450,6 +451,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
     {"response.completed", "response.incomplete", "response.failed"}
 )
 REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
+REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally"
 
 # SSL/TLS cipher configuration for faster handshakes
 # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
@@ -675,7 +677,7 @@ request_timeout: float = (
     DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env
 )
 request_timeout_explicitly_set: bool = _request_timeout_from_env is not None
-DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)  # 10 minutes
+DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)
 AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0
 AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000
 # Patterns that indicate a localhost/internal URL in A2A agent cards that should be
@@ -698,6 +700,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
 STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
 STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
+STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
+STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
+VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
     "DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400
@@ -967,7 +972,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
 
 OPENAI_TRANSCRIPTION_PARAMS: Final = [
     "language",
+    "languages",
+    "keywords",
     "response_format",
+    "stream",
     "timestamp_granularities",
 ]
 

litellm/cost_calculator.py

@@ -6,6 +6,7 @@ import time
 from collections.abc import Mapping, Sequence
 from datetime import datetime
 from functools import lru_cache
+from itertools import accumulate
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
@@ -22,9 +23,14 @@ from litellm.constants import (
     OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
     REALTIME_METERED_USAGE_KEY,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
+from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
-from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
+from litellm.litellm_core_utils.decisions_pricing import (
+    decisions_pricing_fields,
+    decisions_request_price,
+    is_decisions_call_type,
+)
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
@@ -319,12 +325,41 @@ def _get_additional_costs(
     return None
 
 
-def _transcription_usage_has_token_details(
+_TranscriptionRateKey: TypeAlias = Literal[
+    "input_cost_per_second",
+    "output_cost_per_second",
+    "input_cost_per_audio_token",
+    "output_cost_per_audio_token",
+    "input_cost_per_token",
+    "output_cost_per_token",
+]
+_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_audio_token",
+    "output_cost_per_audio_token",
+)
+_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
+    "input_cost_per_second",
+    "output_cost_per_second",
+)
+
+
+def _transcription_uses_token_pricing(
+    model: str,
+    custom_llm_provider: str | None,
     usage_block: Usage | None,
 ) -> bool:
     if usage_block is None:
         return False
 
+    model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
+    if (
+        _declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
+        and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
+    ):
+        return False
+
     prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
     completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
     prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
@@ -338,13 +373,38 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
-def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
-    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
+    if pricing_model is None or custom_llm_provider is None:
         return False
+    transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
+    transcription_metadata: Final = (
+        as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
+    )
+    return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
+
+
+def _transcription_duration_for_cost(
+    completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
+) -> float:
+    hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
+    measured_duration: Final = normalized_audio_duration_seconds(
+        hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
+    )
+    if measured_duration is not None and measured_duration > 0:
+        return measured_duration
+    provider_duration: Final = _get_transcription_usage_duration(completion_response)
+    if (
+        provider_duration is not None
+        and provider_duration > 0
+        and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
+    ):
+        return provider_duration
+    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+        return 0.0
     transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
         model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
     )
-    return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
+    return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
 
 
 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@@ -604,6 +664,9 @@ def cost_per_token(
         decisions_overrides: Final = decisions_pricing_fields(
             litellm.model_cost.get(decisions_model_info["key"], {})  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # model cost map is dynamically loaded
         )
+        request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides)
+        if request_price is not None:
+            return request_price, 0.0
         if decisions_overrides:
             return generic_cost_per_token(
                 model=model,
@@ -693,7 +756,7 @@ def cost_per_token(
             vertex_location=vertex_location,
         )
     elif call_type == "atranscription" or call_type == "transcription":
-        if _transcription_usage_has_token_details(usage_block):
+        if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
             return generic_cost_per_token(
                 model=model,
                 usage=usage_block,
@@ -1275,6 +1338,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
     )
 
 
+def _get_transcription_usage_duration(completion_response: object) -> float | None:
+    response_fields: Final = as_str_mapping(completion_response)
+    usage_object: Final = (
+        response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
+    )
+    usage_fields: Final = as_str_mapping(usage_object)
+    usage_type: Final = (
+        usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
+    )
+    if usage_type != "duration":
+        return None
+    seconds: Final = (
+        usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
+    )
+    return normalized_audio_duration_seconds(seconds)
+
+
 def _extraction_usage(usage: ExtractionUsage) -> Usage:
     return Usage(
         prompt_tokens=usage.input_tokens,
@@ -2000,20 +2080,8 @@ def completion_cost(
                         prompt,
                     )
                 elif call_type in _TRANSCRIPTION_CALL_TYPES:
-                    # Check _hidden_params first (duration stored there to
-                    # avoid polluting the response body), then fall back to
-                    # the response attribute (for verbose_json responses that
-                    # naturally include duration from the provider).
-                    _hidden = getattr(completion_response, "_hidden_params", {}) or {}
-                    audio_transcription_file_duration = _hidden.get(
-                        "audio_transcription_duration",
-                        getattr(completion_response, "duration", 0.0),
-                    ) or (
-                        completion_response.usage.seconds
-                        if isinstance(completion_response, TranscriptionResponse)
-                        and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
-                        and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
-                        else 0.0
+                    audio_transcription_file_duration = _transcription_duration_for_cost(
+                        completion_response, model, requested_model, custom_llm_provider
                     )
                 elif call_type in _RERANK_CALL_TYPES:
                     if completion_response is not None and isinstance(completion_response, RerankResponse):
@@ -2123,6 +2191,7 @@ def completion_cost(
                         litellm_logging_obj=litellm_logging_obj,
                         custom_pricing_model=selected_model if custom_pricing else None,
                         base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
+                        deployment_models=(base_model, requested_model),
                     )
                 elif call_type == _MCP_CALL_TYPE:
                     from litellm.proxy.mcp_server.cost_calculator import (
@@ -3400,8 +3469,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
     def create_logging_realtime_object(
         usage: Usage, results: OpenAIRealtimeStreamList
     ) -> LiteLLMRealtimeStreamLoggingObject:
+        translated_input_seconds, _ = _translation_session_seconds(results)
         return LiteLLMRealtimeStreamLoggingObject(
-            usage=usage,
+            usage=usage.model_copy(update={"seconds": translated_input_seconds})
+            if translated_input_seconds > 0
+            else usage,
             results=results,
         )
 
@@ -3486,6 +3558,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
+_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
 
 
 class _LiveResponsePayload(BaseModel):
@@ -3708,6 +3781,7 @@ def handle_realtime_stream_cost_calculation(
     litellm_logging_obj: LitellmLoggingObject | None = None,
     custom_pricing_model: str | None = None,
     base_pricing_model: str | None = None,
+    deployment_models: Sequence[str | None] = (),
 ) -> float:
     """
     Handles the cost calculation for realtime stream responses.
@@ -3752,6 +3826,13 @@ def handle_realtime_stream_cost_calculation(
         if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
         else 0.0
     )
+    translation_cost: Final = handle_realtime_translation_cost_calculation(
+        results=results,
+        custom_llm_provider=custom_llm_provider,
+        litellm_model_name=litellm_model_name,
+        deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
+        session_model=received_model,
+    )
     live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
     live_session_started: Final = any(result["type"] == "session.started" for result in results)
     live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
@@ -3808,12 +3889,19 @@ def handle_realtime_stream_cost_calculation(
     input_cost: Final = input_cost_per_token + live_response_input_cost
     output_cost: Final = output_cost_per_token + live_response_output_cost
     total_cost: Final = (
-        input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
+        input_cost
+        + output_cost
+        + transcription_cost
+        + translation_cost
+        + live_voice_cost
+        + audio_duration_cost
+        + text_message_cost
     )
     additional_costs: Final = tuple(
         (cost_name, cost)
         for cost_name, cost in (
             ("transcription_cost", transcription_cost),
+            ("translation_cost", translation_cost),
             ("live_voice_cost", live_voice_cost),
             ("audio_duration_cost", audio_duration_cost),
             ("text_message_cost", text_message_cost),
@@ -3834,6 +3922,55 @@ def handle_realtime_stream_cost_calculation(
     return total_cost
 
 
+def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
+    usage_events: Final = tuple(
+        usage
+        for result in results
+        if (event := as_str_mapping(result)) is not None
+        and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
+        and (usage := as_str_mapping(event.get("usage"))) is not None
+    )
+    input_seconds: Final = sum(
+        seconds
+        for usage in usage_events
+        if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
+    )
+    output_seconds: Final = sum(
+        output_duration
+        for usage in usage_events
+        if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
+    )
+    return float(input_seconds), float(output_seconds)
+
+
+def _translation_per_second_rates(
+    models: Sequence[str | None], custom_llm_provider: str
+) -> tuple[float | None, float | None]:
+    model_infos: Final = tuple(
+        _lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
+    )
+    return (
+        _first_declared_transcription_rate(("input_cost_per_second",), model_infos),
+        _first_declared_transcription_rate(("output_cost_per_second",), model_infos),
+    )
+
+
+def handle_realtime_translation_cost_calculation(
+    results: OpenAIRealtimeStreamList,
+    custom_llm_provider: str,
+    litellm_model_name: str,
+    deployment_pricing_models: Sequence[str | None] = (),
+    session_model: str | None = None,
+) -> float:
+    input_seconds, output_seconds = _translation_session_seconds(results)
+    if input_seconds <= 0 and output_seconds <= 0:
+        return 0.0
+    input_rate, output_rate = _translation_per_second_rates(
+        (*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
+    )
+    return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
+
+
 def handle_realtime_transcription_cost_calculation(
     results: OpenAIRealtimeStreamList,
     custom_llm_provider: str,
@@ -3849,57 +3986,77 @@ def handle_realtime_transcription_cost_calculation(
       - {"type": "duration", "seconds": <float>}  → priced via input_cost_per_second
       - {"type": "tokens", "input_tokens": ...}    → priced via input/audio token cost
     """
-    completed_events: Final = [
-        cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
-    ]
-    if not completed_events:
+    configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
+    clips: Final = tuple(
+        (as_str_mapping(result.get("usage")) or {}, configured)
+        for result, configured in zip(results, configured_models, strict=True)
+        if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
+    )
+    if not clips:
         return 0.0
 
-    model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
-    model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
+    fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
+    clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
+    model_infos: Final = MappingProxyType(
+        {model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
+    )
     override_info: Final = (
         _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
     )
+    return sum(
+        (_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
+        0.0,
+    )
 
-    total_cost = 0.0
-    for event in completed_events:
-        usage = event.get("usage") or {}
-        total_cost += _transcription_usage_cost(usage, model_info, override_info)
-    return total_cost
+
+_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
+    {"transcription_session.created", "transcription_session.updated", "session.created", "session.updated"}
+)
+
+
+def _str_field(mapping: Mapping[str, object] | None, key: str) -> str | None:
+    value: Final = None if mapping is None else mapping.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _configured_transcription_model(session: Mapping[str, object]) -> str | None:
+    audio_input: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("input"))
+    nested: Final = _str_field(as_str_mapping((audio_input or {}).get("transcription")), "model")
+    return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
+
+
+def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
+    return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
+
+
+def _session_transcription_model(result: Mapping[str, object]) -> str | None:
+    session: Final = _session_of(result)
+    return None if session is None else _configured_transcription_model(session)
+
+
+def _newer_model(current: str | None, newer: str | None) -> str | None:
+    return newer or current
 
 
 def _get_transcription_model_name_from_results(
     results: OpenAIRealtimeStreamList,
 ) -> str | None:
-    """Resolve the ASR model from a transcription_session.* / session.* event."""
-    for result in results:
-        if result.get("type") in (
-            "transcription_session.created",
-            "transcription_session.updated",
-            "session.created",
-            "session.updated",
-        ):
-            session = cast(dict, result).get("session", {}) or {}
-            transcription = ((session.get("audio", {}) or {}).get("input", {}) or {}).get(
-                "transcription", {}
-            ) or session.get("input_audio_transcription", {})
-            model = (transcription or {}).get("model") or session.get("model")
-            if model:
-                return model
-    return None
+    return next(
+        (model for result in results if (model := _str_field(_session_of(result), "model"))),
+        None,
+    )
 
 
-_TranscriptionRateKey: TypeAlias = Literal[
-    "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
-]
-
-
-def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
+def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
     match key:
         case "input_cost_per_second":
             return info.get("input_cost_per_second")
+        case "output_cost_per_second":
+            return info.get("output_cost_per_second")
         case "input_cost_per_audio_token":
             return info.get("input_cost_per_audio_token")
+        case "output_cost_per_audio_token":
+            return info.get("output_cost_per_audio_token")
         case "input_cost_per_token":
             return info.get("input_cost_per_token")
         case "output_cost_per_token":
@@ -3908,7 +4065,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
             assert_never(key)
 
 
-def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
+def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
     """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
     because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
     if info is None:
@@ -3926,15 +4083,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
     )
 
 
-def _transcription_rate(
-    keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
-) -> float:
-    rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
-    return next((rate for rate in rates if rate is not None), 0.0)
+def _first_declared_transcription_rate(
+    keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
+) -> float | None:
+    rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
+    return next((rate for rate in rates if rate is not None), None)
+
+
+def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
+    return _first_declared_transcription_rate(keys, infos) or 0.0
 
 
 def _transcription_usage_cost(
-    usage: dict,
+    usage: Mapping[str, object],
     model_info: ModelInfo | None,
     override_info: ModelInfo | None = None,
 ) -> float:
@@ -3943,13 +4104,13 @@ def _transcription_usage_cost(
 
     usage_type: Final = usage.get("type")
     if usage_type == "duration":
-        seconds: Final = usage.get("seconds") or 0.0
+        seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
         return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
     if usage_type == "tokens":
-        input_token_details: Final = usage.get("input_token_details") or {}
-        audio_tokens: Final = input_token_details.get("audio_tokens") or 0
-        text_tokens: Final = input_token_details.get("text_tokens") or 0
-        output_tokens: Final = usage.get("output_tokens") or 0
+        input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
+        audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
+        text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
+        output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
         audio_cost: Final = float(audio_tokens) * _transcription_rate(
             ("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
         )

litellm/decisions/call.py

@@ -13,6 +13,7 @@ from typing_extensions import assert_never
 import litellm
 from litellm.cost_calculator import cost_map_key_prices_anything
 from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
+from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
 from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
@@ -555,7 +556,7 @@ def prepare_call(
         resolved_api_base, static_api_key = provider_config.resolve_connection(
             dynamic_api_base or call_api_base, dynamic_api_key or api_key
         )
-    except ValueError as error:
+    except (ValueError, httpx.InvalidURL) as error:
         raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
     if resolved_api_base is None:
         raise litellm.BadRequestError(
@@ -613,6 +614,9 @@ def prepare_call(
             litellm_params={
                 "litellm_call_id": kwargs.get("litellm_call_id"),
                 "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
+                "input_cost_per_query": (
+                    kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
+                ),
             },
             custom_llm_provider=provider,
         )
@@ -787,7 +791,9 @@ def _formatted_response(
         case DecisionsRequestBody():
             return ir_to_systemone_response(caller_response, caller_request)
         case OpenAIDecisionRequestBody():
-            return ir_to_openai_response(caller_response, caller_request, call.requested_model)
+            return ir_to_openai_response(
+                caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
+            )
         case _:
             assert_never(call.request)
 

litellm/guardrails.py

@@ -0,0 +1,66 @@
+"""Invoke configured gateway guardrails without generating a completion."""
+
+from collections.abc import Mapping, Sequence
+from enum import Enum
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # shared factory has untyped optional parameters
+)
+from litellm.types.guardrails import NativeGuardrailResponse
+
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class _InputType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+async def apply(
+    guardrail_name: str,
+    *,
+    input: str | None = None,
+    response: str | None = None,
+    messages: Sequence[Mapping[str, object]] | None = None,
+    metadata: Mapping[str, object] | None = None,
+    api_base: str | None = None,
+    api_key: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+) -> NativeGuardrailResponse:
+    """Return a configured guardrail's native structured findings.
+
+    Supply exactly one of ``input`` or ``response``. Grounding guardrails can
+    use ``messages`` as evidence. Connection settings default to
+    ``litellm.api_base`` and ``litellm.api_key``.
+    """
+    if (input is None) == (response is None):
+        raise ValueError("Supply exactly one of input or response.")
+    resolved_base: Final = api_base if api_base is not None else litellm.api_base
+    if not resolved_base:
+        raise ValueError("Set api_base or litellm.api_base to your LiteLLM gateway URL.")
+    resolved_key: Final = api_key if api_key is not None else litellm.api_key
+    input_type: Final = _InputType.RESPONSE if response is not None else _InputType.REQUEST
+    payload: Final = dict[str, object](
+        guardrail_name=guardrail_name,
+        text=response if response is not None else input,
+        input_type=input_type.value,
+        response_format="native",
+    )
+    if messages is not None:
+        payload["messages"] = [dict(message) for message in messages]
+    if metadata is not None:
+        payload["metadata"] = dict(metadata)
+    gateway_root: Final = resolved_base.rstrip("/").removesuffix("/v1")
+    client: Final = get_async_httpx_client(llm_provider="litellm_guardrails")
+    result: Final = await client.post(  # pyright: ignore[reportUnknownMemberType]  # shared handler has untyped optional parameters
+        url=f"{gateway_root}/guardrails/apply_guardrail",
+        json=payload,
+        headers={"Authorization": f"Bearer {resolved_key}"} if resolved_key else {},
+        timeout=timeout,
+    )
+    return _NATIVE_RESPONSE.validate_json(result.content)

litellm/images/main.py

@@ -28,8 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
-from litellm.llms.custom_llm import CustomLLM
-from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
+from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params
 
 #################### Initialize provider clients ####################
 llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@@ -608,10 +607,7 @@ def image_generation(
             )
         elif custom_llm_provider in litellm._custom_providers:  # Assume custom LLM provider
             # Get the Custom Handler
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -915,10 +911,7 @@ def image_edit(
 
         # Check for custom provider
         if custom_llm_provider in litellm._custom_providers:
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -995,10 +988,15 @@ def image_edit(
             or custom_llm_provider == "azure"
             or custom_llm_provider in litellm.openai_compatible_providers
         ):
+            dropped_form_params: Final = header_auth_params(custom_llm_provider)
             image_edit_request_params.update(
                 flatten_form_field_values(
-                    non_default_params,
-                    extra_body if isinstance(extra_body, dict) else None,
+                    {key: value for key, value in non_default_params.items() if key not in dropped_form_params},
+                    (
+                        {key: value for key, value in extra_body.items() if key not in dropped_form_params}
+                        if isinstance(extra_body, dict)
+                        else None
+                    ),
                 )
             )
 

litellm/integrations/anthropic_cache_control_hook.py

@@ -9,29 +9,35 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
 
 """
 
+import bisect
 import copy
 import itertools
 import os
 import re
-from collections.abc import Callable, Container, Iterable, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
+from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
-from pydantic import TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import PromptManagementClient
-from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
     cache_control_as_prompt_cache_breakpoint,
+    prompt_cache_breakpoint_as_cache_control,
     with_prompt_cache_breakpoint,
+    without_prompt_cache_breakpoints,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     tool_call_ids_answered_by_client,
     tool_call_ids_answered_in,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
 from litellm.llms.anthropic.common_utils import (
     is_claude_code_one_shot_subagent_request,
     supports_anthropic_cache_control,
@@ -72,7 +78,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
 
 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
 OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
-_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
+_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+)|-(0|[1-9]\d?)(?![\da-z]))?")
 _FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
 _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
 OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
@@ -84,8 +90,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
 _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
 _CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object])
 _ATTRIBUTE_VALUE: Final = TypeAdapter(object)
+_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 
 AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
+CostMap: TypeAlias = Mapping[str, Mapping[str, object]]
+
+
+def routed_model_info(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MODEL_INFO_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap:
+    import litellm
+
+    if model_cost is not None:
+        return model_cost
+    return cast(CostMap, litellm.model_cost)  # cast-ok: the live map holds catalog entries
+
+
+def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None:
+    declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
+    return declared if isinstance(declared, bool) else None
+
+
+def routed_cost_map() -> CostMap:
+    from litellm.utils import catalog_with_runtime_registrations
+
+    return catalog_with_runtime_registrations()
+
+
+def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None:
+    return None if model_info is None else routed_cost_map()
 
 
 def _validated_object_mapping(value: object) -> dict[object, object] | None:
@@ -128,7 +166,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
         return False
     return (
         not isinstance(fields.get("content"), list)
-        or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
+        or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
     )
 
 
@@ -294,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object:
     return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
 
 
+def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object:
+    if not _carries_prompt_cache_breakpoint(part):
+        return part
+    return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params)
+
+
 def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
     if not isinstance(value, (list, tuple)):
         return ()
@@ -313,19 +357,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non
     return provider
 
 
-def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
-    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
+def supports_openai_prompt_cache_breakpoint(
+    model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool:
+    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True
 
 
-def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
+def openai_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool | None:
+    declared: Final = declared_prompt_cache_breakpoint(model_info)
+    if declared is not None:
+        return declared
+    model_cost: Final = _catalog_unless_unrouted(model_info)
     hosted_flag: Final = (
         None
         if custom_llm_provider is None
-        else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model)
+        else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost)
     )
     if hosted_flag is not None:
         return hosted_flag
-    model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
+    model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost)
     if model_map_flag is not None:
         return model_map_flag
     fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
@@ -334,6 +386,35 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str
     return _named_model_breakpoint_support(model)
 
 
+class _DeclaredBaseModels(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+
+    base_model: object = None
+    model_info: object = None
+
+    def names(self) -> tuple[str, ...]:
+        model_info: Final = _validated_object_mapping(self.model_info) or {}
+        candidates: Final = (self.base_model, model_info.get("base_model"))
+        return tuple(name for name in candidates if isinstance(name, str) and name)
+
+
+def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None:
+    return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info)
+
+
+def deployment_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str, litellm_params: object
+) -> bool | None:
+    """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
+    declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
+    model_info: Final = routed_model_info(declared.model_info)
+    supports: Final = (
+        openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info)
+        for name in dict.fromkeys((*declared.names(), model))
+    )
+    return next((support for support in supports if support is not None), None)
+
+
 def _named_model_breakpoint_support(model: str) -> bool | None:
     bare_model: Final = model.rsplit("/", 1)[-1].lower()
     if _O_SERIES_PATTERN.match(bare_model) is not None:
@@ -341,15 +422,14 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
     version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
     if version_match is None:
         return None
-    version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
+    version: Final = (int(version_match.group(1)), int(version_match.group(2) or version_match.group(3) or 0))
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 
 
-def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
-    import litellm
-
-    entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
-    flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
+def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None:
+    cost_map: Final = _live_cost_map_unless(model_cost)
+    entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1]))
+    flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping))
     return next((flag is True for flag in flags if flag is not None), None)
 
 
@@ -365,7 +445,7 @@ def _hosted_openai_dialect_flag(
     model: str,
     custom_llm_provider: str | None,
     resolve_provider: Callable[[str], str | None],
-    model_cost: Mapping[str, Mapping[str, object]] | None = None,
+    model_cost: CostMap | None = None,
 ) -> bool | None:
     """
     Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@@ -382,13 +462,7 @@ def _hosted_openai_dialect_flag(
     row of the same name, so an exact entry only speaks for a deployment when it is
     keyed for that deployment's provider.
     """
-    import litellm
-
-    cost_map: Final = (
-        cast(Mapping[str, Mapping[str, object]], litellm.model_cost)  # cast-ok: the live map holds catalog entries
-        if model_cost is None
-        else model_cost
-    )
+    cost_map: Final = _live_cost_map_unless(model_cost)
     exact_entry: Final = cost_map.get(model)
     exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
     if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
@@ -398,10 +472,16 @@ def _hosted_openai_dialect_flag(
         return None
     if isinstance(exact_entry, Mapping) and exact_provider == provider:
         return _hosted_entry_flag(exact_entry, lambda: provider)
+    entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider))
+    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
+    return next((flag for flag in flags if flag is not None), None)
+
+
+def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]:
     from litellm.utils import get_potential_model_names
 
     names: Final = get_potential_model_names(model, provider)
-    candidates: Final = (
+    return (
         names["combined_model_name"],
         names["region_free_combined_model_name"],
         names["split_model"],
@@ -409,9 +489,60 @@ def _hosted_openai_dialect_flag(
         names["stripped_model_name"],
         names["provider_prefixed_model_name"],
     )
-    entries: Final = (cost_map.get(candidate) for candidate in candidates)
-    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
-    return next((flag for flag in flags if flag is not None), None)
+
+
+def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool:
+    entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider)))
+    return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries)
+
+
+_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse"))
+_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle"))
+
+
+def _declared_hosted_flag(
+    declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object
+) -> bool | None:
+    """
+    A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the
+    request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes
+    a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep
+    the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass
+    the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost
+    map, or the deployment's own ``litellm_provider``, lists under that provider.
+    """
+    if provider is None or provider == "openai":
+        return None
+    return (
+        declared
+        and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT
+        and (
+            provider in _PROVIDERS_HOSTING_OPENAIS_API
+            or deployment_provider == provider
+            or _cost_map_lists_model_for_provider(model, provider, model_cost)
+        )
+        and _sends_openai_shaped_chat(model, provider)
+    )
+
+
+@lru_cache(maxsize=512)
+def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool:
+    from litellm.exceptions import BadRequestError
+    from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
+    from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig
+    from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+    from litellm.llms.openai.openai import OpenAIConfig
+    from litellm.types.utils import LlmProviders
+    from litellm.utils import ProviderConfigManager
+
+    try:
+        served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider)
+        chat_config: Final = ProviderConfigManager.get_provider_chat_config(
+            model=served_model, provider=LlmProviders(provider)
+        )
+    except (BadRequestError, ValueError):
+        return False
+    return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig))
 
 
 def targets_openai_api(api_base: object) -> bool:
@@ -431,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool:
     return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS)
 
 
+def _carries_cache_control(block: object) -> bool:
+    return _attribute_or_key(block, "cache_control") is not None
+
+
 def _attribute_or_key(value: object, key: str) -> object | None:
     if hasattr(value, key):
         return _ATTRIBUTE_VALUE.validate_python(getattr(value, key))
@@ -445,22 +580,41 @@ def _as_object_list(value: object | None) -> list[object] | None:
     return _validated_object_list(value)
 
 
-def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, answered_tool_ids: Container[str]) -> bool:
+def _server_tool_call_id(tool_call: object) -> str | None:
+    tool_call_id: Final = _attribute_or_key(tool_call, "id")
+    if isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
+        return tool_call_id
+    return None
+
+
+def _tool_call_carries_cache_breakpoint(tool_call: object, client_tool_call_ids: Container[str]) -> bool:
     if _attribute_or_key(tool_call, "cache_control") is None:
         return False
+    server_tool_call_id: Final = _server_tool_call_id(tool_call)
+    return server_tool_call_id is None or server_tool_call_id in client_tool_call_ids
 
-    return not tool_call_is_rebuilt_as_server_tool_use(
-        _attribute_or_key(tool_call, "id"), _attribute_or_key(message, "provider_specific_fields"), answered_tool_ids
+
+def _server_tool_call_ids_without_a_kept_result(message: object) -> Iterator[str]:
+    provider_specific_fields: Final = _attribute_or_key(message, "provider_specific_fields")
+    tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or ()
+    return (
+        tool_call_id
+        for tool_call in tool_calls
+        if (tool_call_id := _server_tool_call_id(tool_call)) is not None
+        and not tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
     )
 
 
-def _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+def _client_tool_call_ids_when_a_server_tool_call_is_marked(
     messages: Sequence[object], model: str | None = None, custom_llm_provider: str | None = None
 ) -> frozenset[str]:
     if not _marks_a_server_tool_call(messages):
         return frozenset()
     if model is not None and not _builds_anthropic_messages(model, custom_llm_provider):
-        return tool_call_ids_answered_in(messages)
+        server_calls_sent_as_client_calls: Final = itertools.chain.from_iterable(
+            map(_server_tool_call_ids_without_a_kept_result, messages)
+        )
+        return tool_call_ids_answered_in(messages) | frozenset(server_calls_sent_as_client_calls)
     return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider))
 
 
@@ -491,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b
     return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig)
 
 
+def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
+    return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages(
+        model, custom_llm_provider
+    )
+
+
 def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
     provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
     return provider if isinstance(provider, str) else None
@@ -517,12 +677,7 @@ def _model_info_flags_mid_conversation_system(model: str, custom_llm_provider: s
 
 
 def _is_marked_server_tool_call(tool_call: object) -> bool:
-    tool_call_id: Final = _attribute_or_key(tool_call, "id")
-    return (
-        isinstance(tool_call_id, str)
-        and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
-        and _attribute_or_key(tool_call, "cache_control") is not None
-    )
+    return _server_tool_call_id(tool_call) is not None and _attribute_or_key(tool_call, "cache_control") is not None
 
 
 def _tool_carries_cache_breakpoint(tool: object) -> bool:
@@ -539,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
     return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
 
 
+def _carries_prompt_cache_breakpoint(block: object) -> bool:
+    return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
+
+
 def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
     return message.get("role") != "assistant"
 
 
+def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
+    return message.get("role") == "tool" and any(
+        _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
+    )
+
+
+def _part_without_prompt_cache_breakpoint(part: object) -> object:
+    if not _carries_prompt_cache_breakpoint(part):
+        return part
+    return {
+        key: value
+        for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items()
+        if key != "prompt_cache_breakpoint"
+    }
+
+
+def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
+    if not _tool_result_carries_prompt_cache_breakpoint(message):
+        return message
+    parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
+    return cast(  # cast-ok: the same tool message with its part marks removed
+        AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]}
+    )
+
+
+def _created_cache_control(original: object, translated: object) -> bool:
+    return (
+        _attribute_or_key(original, "cache_control") is None
+        and _attribute_or_key(translated, "cache_control") is not None
+    )
+
+
+def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]:
+    if translated is original:
+        return ()
+    original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or []
+    translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or []
+    return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts))
+
+
+def _part_without_cache_control(part: object) -> object:
+    return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"}
+
+
+def _message_dropping_created_cache_controls_between(
+    message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int
+) -> AllMessageValues:
+    if not any(created):
+        return message
+    parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
+    ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before)
+    return cast(  # cast-ok: the same message with its middle translated marks removed
+        AllMessageValues,
+        {
+            **message,
+            "content": [
+                _part_without_cache_control(part)
+                if is_created and first_dropped <= ordinal < first_kept_after
+                else part
+                for part, is_created, ordinal in zip(parts, created, ordinals)
+            ],
+        },
+    )
+
+
 # Set by a caller whose message list is not the one that goes upstream -- today the
 # Responses API layer, whose `instructions` only becomes a system message further down.
 # Tells this hook to hand role-targeted points to the pass holding the final messages
@@ -628,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if not injection_points:
             return model, messages, non_default_params
 
-        # Create a deep copy of messages to avoid modifying the original list
-        processed_messages = copy.deepcopy(messages)
-
         message_points: Final = tuple(
             cast(CacheControlMessageInjectionPoint, point)
             for point in injection_points
@@ -638,18 +859,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message")
 
-        custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0])
+        request_params: Final = cast(Mapping[str, object], non_default_params)  # cast-ok: base hook's bare dict
+        custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
         stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
+        model_info: Final = routed_model_info(request_params.get("model_info"))
         openai_dialect: Final = (
             stamped_dialect
             if isinstance(stamped_dialect, bool)
             else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
                 model,
                 custom_llm_provider,
-                non_default_params.get("api_base") or non_default_params.get("base_url"),
-                non_default_params.get("prompt_cache_options"),
+                request_params.get("api_base") or request_params.get("base_url"),
+                request_params.get("prompt_cache_options"),
+                model_info,
             )
         )
+        processed_messages = copy.deepcopy(
+            [without_prompt_cache_breakpoints(message) for message in messages]
+            if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
+            else messages
+        )
         # A provisional message list defers every role-targeted point to the pass holding
         # the final one: a role with no message here may have one there, and settling all
         # of them in one pass is what lets config order decide the shared breakpoint
@@ -668,7 +897,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
             remaining_points, external_breakpoints, openai_dialect
         )
-        answered_tool_ids: Final = _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+        answered_tool_ids: Final = _client_tool_call_ids_when_a_server_tool_call_is_marked(
             processed_messages, model, custom_llm_provider
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(
@@ -718,32 +947,35 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object = None,
         prompt_cache_options: object = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> bool:
         if model is None:
             return False
-        hosted_flag: Final = _hosted_openai_dialect_flag(
-            model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
+        declared: Final = declared_prompt_cache_breakpoint(model_info)
+        hosted_flag: Final = (
+            _hosted_openai_dialect_flag(
+                model,
+                custom_llm_provider,
+                lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base),
+                _catalog_unless_unrouted(model_info),
+            )
+            if declared is None
+            else _declared_hosted_flag(
+                declared,
+                model,
+                custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base),
+                routed_cost_map(),
+                None if model_info is None else model_info.get("litellm_provider"),
+            )
         )
         if hosted_flag is not None:
-            return hosted_flag
-        if not supports_openai_prompt_cache_breakpoint(model):
+            return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
+        if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
             return False
         if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
             return False
         return prompt_cache_options is not None or targets_openai_api(api_base)
 
-    @staticmethod
-    def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
-        return (
-            _hosted_openai_dialect_flag(
-                model,
-                custom_llm_provider,
-                AnthropicCacheControlHook._resolve_provider,
-                GetModelCostMap.loaded_model_cost_map(),
-            )
-            is True
-        )
-
     @staticmethod
     def _resolve_provider(model: str, api_base: object = None) -> str | None:
         return _served_provider_for_model(model, api_base)
@@ -757,7 +989,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         message_list: Final = tuple(messages)
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(message_list)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(message_list)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -790,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
 
+    @staticmethod
+    def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]:
+        return tuple(
+            tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool)
+        )
+
+    @staticmethod
+    def count_chat_external_cache_breakpoints(optional_params: object) -> int:
+        params: Final = _validated_object_mapping(optional_params) or {}
+        return AnthropicCacheControlHook.count_external_cache_breakpoints(
+            _as_object_list(params.get("tools")), params.get("cache_control")
+        )
+
     @staticmethod
     def count_external_cache_breakpoints_on_messages_route(
         tools: Iterable[object] | None, cache_control: object, request_kwargs: object
@@ -849,7 +1094,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         so earlier points win when slots are scarce.
         """
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(messages)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -939,16 +1184,22 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
         tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
         tool_call_count: Final = (
-            sum(
-                1
-                for tool_call in tool_calls
-                if _tool_call_carries_cache_breakpoint(tool_call, message, answered_tool_ids)
-            )
+            sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, answered_tool_ids))
             if tool_calls
             else 0
         )
         return message_count + content_count + tool_call_count
 
+    @staticmethod
+    def count_converse_cache_points(message: AllMessageValues) -> int:
+        content: Final = _as_object_list(_attribute_or_key(message, "content"))
+        tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or []
+        marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block))
+        marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call))
+        if message.get("role") == "tool":
+            return int(_carries_cache_control(message) or marked_parts > 0)
+        return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls
+
     @staticmethod
     def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool:
         """Return True if the message already carries any cache_control."""
@@ -1007,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             return renamed
         return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
 
+    @staticmethod
+    def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
+        """Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's
+        ``cache_control``. Marks with no OpenAI slot are left for the caller to strip."""
+        if not _may_carry_prompt_cache_breakpoint(message):
+            return message
+        parts: Final = _as_object_list(_attribute_or_key(message, "content"))
+        if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts):
+            return message
+        return cast(  # cast-ok: the same message with its part marks renamed
+            AllMessageValues,
+            {**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]},
+        )
+
+    @staticmethod
+    def prompt_cache_breakpoints_dropped_from_tool_results(
+        messages: Sequence[AllMessageValues], drop_params: bool, model: str
+    ) -> tuple[AllMessageValues, ...]:
+        if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages):
+            return tuple(messages)
+        if not drop_params:
+            raise UnsupportedParamsError(
+                message=(
+                    f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool "
+                    f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
+                ),
+                status_code=400,
+            )
+        verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model)
+        return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages)
+
+    @staticmethod
+    def prompt_cache_breakpoints_as_capped_cache_control(
+        messages: Sequence[AllMessageValues],
+        drop_params: bool,
+        sent_breakpoints: Callable[[AllMessageValues], int],
+        external_breakpoints: Callable[[], int],
+    ) -> tuple[AllMessageValues, ...]:
+        """``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's
+        breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends.
+        Under ``drop_params`` the first and last translated marks fill the slots the client's own
+        ``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest
+        turns stay cached, and without it a request over the cap is rejected."""
+        translated: Final = tuple(
+            AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params)
+            for message in messages
+        )
+        if all(after is before for before, after in zip(messages, translated)):
+            return translated
+        created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated))
+        created_marks: Final = sum(sum(flags) for flags in created)
+        if created_marks == 0:
+            return translated
+        external: Final = external_breakpoints()
+        breakpoints: Final = sum(map(sent_breakpoints, translated)) + external
+        excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS
+        if excess <= 0:
+            return translated
+        if not drop_params:
+            raise UnsupportedParamsError(
+                message=(
+                    f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this "
+                    f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} "
+                    f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them."
+                ),
+                status_code=400,
+            )
+        verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params")
+        marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0))
+
+        def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]:
+            first_dropped: Final = (kept_marks + 1) // 2
+            first_kept_after: Final = created_marks - kept_marks // 2
+            return tuple(
+                _message_dropping_created_cache_controls_between(
+                    message, flags, before, first_dropped, first_kept_after
+                )
+                for message, flags, before in zip(translated, created, marks_before)
+            )
+
+        def over_cap(kept_marks: int) -> bool:
+            return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS
+
+        first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap)
+        return capped_request(max(0, first_over_cap - 1))
+
     @staticmethod
     def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
         if not _may_carry_prompt_cache_breakpoint(message):
@@ -1146,6 +1483,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object,
         prompt_cache_options: object,
+        model_info: Mapping[str, object] | None = None,
     ) -> Sequence[Mapping[str, object]]:
         """Carry onto the points what the prompt-management hook never receives.
 
@@ -1157,7 +1495,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         across requests.
         """
         with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect(
-            points, model, custom_llm_provider, api_base, prompt_cache_options
+            points, model, custom_llm_provider, api_base, prompt_cache_options, model_info
         )
         stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect)
         if external_breakpoints == 0 and not stamped_earlier:
@@ -1171,14 +1509,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object,
         prompt_cache_options: object,
+        model_info: Mapping[str, object] | None = None,
     ) -> Sequence[Mapping[str, object]]:
-        if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider):
+        if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(
+            model, custom_llm_provider
+        ):
             return points
         return AnthropicCacheControlHook._stamped(
             points,
             "_litellm_openai_dialect",
             AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-                model, custom_llm_provider, api_base, prompt_cache_options
+                model, custom_llm_provider, api_base, prompt_cache_options, model_info
             ),
         )
 
@@ -1237,6 +1578,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         cache_control: object = None,
         request_kwargs: object = None,
         on_messages_route: bool = False,
+        model_info: Mapping[str, object] | None = None,
     ) -> list[CacheControlInjectionPoint]:
         """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
 
@@ -1254,7 +1596,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True:
             return []
 
-        if not supports_anthropic_cache_control(model, custom_llm_provider):
+        if not supports_anthropic_cache_control(model, custom_llm_provider, model_info):
             return []
 
         if AnthropicCacheControlHook._request_has_cache_control(
@@ -1332,6 +1674,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         tools: list | None = None,
         enable_prompt_caching: bool | None = None,
         api_base: object = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> None:
         """For /chat/completions: resolve the injection points the request should carry.
 
@@ -1346,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
 
         configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points"))
         if configured:
-            tools_keeping_marks: Final = tuple(
-                tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool)
-            )
+            tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools)
             stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook(
                 configured,
                 AnthropicCacheControlHook.count_external_cache_breakpoints(
@@ -1358,6 +1699,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 custom_llm_provider,
                 api_base,
                 non_default_params.get("prompt_cache_options"),
+                model_info,
             )
             non_default_params["cache_control_injection_points"] = (
                 AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider)
@@ -1374,6 +1716,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             enable_prompt_caching=enable_prompt_caching,
             cache_control=non_default_params.get("cache_control"),
             request_kwargs=non_default_params,
+            model_info=model_info,
         )
         if points:
             non_default_params["cache_control_injection_points"] = points
@@ -1492,6 +1835,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 cache_control=cache_control,
                 request_kwargs=kwargs,
                 on_messages_route=True,
+                model_info=routed_model_info(kwargs.get("model_info")),
             )
             if model is not None
             else ()
@@ -1500,7 +1844,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             return messages, system
 
         openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-            model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
+            model,
+            custom_llm_provider,
+            api_base,
+            kwargs.get("prompt_cache_options"),
+            routed_model_info(kwargs.get("model_info")),
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
         messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
@@ -1634,9 +1982,17 @@ def messages_within_cache_control_cap(
 
     Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
     """
-    reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
-        _validated_object_list(tools), cache_control, request_kwargs
+    return messages_within_reserved_cache_control_cap(
+        messages,
+        AnthropicCacheControlHook.count_external_cache_breakpoints(
+            _validated_object_list(tools), cache_control, request_kwargs
+        ),
     )
+
+
+def messages_within_reserved_cache_control_cap(
+    messages: Sequence[_MessageT], reserved_marks: int
+) -> Sequence[_MessageT]:
     kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
     dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
     if dropped > 0:

litellm/integrations/azure_storage/azure_storage.py

@@ -1,7 +1,8 @@
 import asyncio
 import os
 import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from functools import cache
 from typing import Final
@@ -15,6 +16,7 @@ from litellm.constants import (
     AZURE_STORAGE_MSFT_VERSION,
 )
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
 from litellm.llms.custom_httpx.http_handler import (
@@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
     return f"{name.translate(_ADLS_SAFE_NAME)}.json"
 
 
+@dataclass(frozen=True, slots=True)
+class AzureStorageLogQueueItem:
+    payload: StandardLoggingPayload
+    file_name: str
+
+
+def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
+    payload_id: Final = payload.get("id")
+    return AzureStorageLogQueueItem(
+        payload=payload,
+        file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
+    )
+
+
 @cache
 def _cached_credential_chain_token_provider() -> Callable[[], str]:
     return get_azure_ad_token_provider(
@@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
 
             asyncio.create_task(self.periodic_flush())
             self.flush_lock = asyncio.Lock()
-            self.log_queue: list[StandardLoggingPayload] = []
+            self.log_queue: list[AzureStorageLogQueueItem] = []
             super().__init__(**kwargs, flush_lock=self.flush_lock)
         except Exception as e:
             verbose_logger.exception(
@@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
     def azure_storage_blob_endpoint(self) -> str:
         return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
 
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
         """
         Async Log success events to Azure Blob Storage
 
@@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             if standard_logging_payload is None:
                 raise ValueError("standard_logging_payload is not set")
 
-            self.log_queue.append(standard_logging_payload)
+            self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
 
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
 
-    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
         """
         Async Log failure events to Azure Blob Storage
 
@@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             if standard_logging_payload is None:
                 raise ValueError("standard_logging_payload is not set")
 
-            self.log_queue.append(standard_logging_payload)
+            self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
 
@@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
                 len(self.log_queue),
             )
 
-            for payload in self.log_queue:
-                await self.async_upload_payload_to_azure_blob_storage(payload=payload)
+            for item in self.log_queue:
+                await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
 
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
 
-    async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
+    async def async_upload_payload_to_azure_blob_storage(
+        self, payload: StandardLoggingPayload, file_name: str | None = None
+    ):
         """
         Uploads the payload to Azure Blob Storage using a 3-step process:
         1. Create file resource
@@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
         """
         try:
             if self.azure_storage_account_key:
-                await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
+                await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
             else:
                 # Get a valid token instead of always requesting a new one
                 await self.set_valid_azure_ad_token()
                 async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
                 json_payload: Final = safe_dumps(payload) + "\n"  # Add newline for each log entry
                 payload_bytes: Final = json_payload.encode("utf-8")
-                filename: Final = adls_safe_file_name(payload.get("id"))
+                filename: Final = file_name or adls_safe_file_name(payload.get("id"))
                 base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
 
                 # Execute the 3-step upload process
@@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
         return self._service_client
 
-    async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
+    async def upload_to_azure_data_lake_with_azure_account_key(
+        self, payload: StandardLoggingPayload, file_name: str | None = None
+    ):
         """
         Uploads the payload to Azure Data Lake using the Azure SDK
 
@@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
 
         try:
             today: Final = datetime.now().strftime("%Y-%m-%d")
-            file_name: Final = adls_safe_file_name(payload.get("id"))
+            resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
             await self._upload_bytes_with_account_key(
-                file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
+                file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
             )
-            verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
+            verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
 
         except Exception as e:
             verbose_logger.exception("Error occurred: %s", e)

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
 from litellm.integrations.clickhouse.context import is_lens_analysis
 from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@@ -134,7 +135,7 @@ def spend_log_row_from_payload(
     usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
     cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
     trace_id, span_id = _find_traceparent(metadata, kwargs)
-    request_id = str(payload.get("id") or "")
+    response_id: Final = str(payload.get("id") or "")
     redact = litellm.turn_off_message_logging is True
     track_end_user: Final = not litellm.disable_end_user_cost_tracking
     completion_start_ms = _to_ms(payload.get("completionStartTime"))
@@ -157,8 +158,8 @@ def spend_log_row_from_payload(
         )
     )
     return SpendLogRecord(
-        request_id=request_id,
-        response_id=strip_cache_hit_suffix(request_id),
+        request_id=logged_event_key(kwargs, response_id),
+        response_id=strip_cache_hit_suffix(response_id),
         provider_request_id=(
             get_provider_request_id(kwargs.get("response_headers"))
             or get_provider_request_id(hidden_params.get("additional_headers"))

litellm/integrations/custom_guardrail.py

@@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
         enrich_http_exception_with_guardrail_context(result, self)
         raise result
 
+    def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
+        return (
+            isinstance(request_data.get("guardrails"), list)
+            and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
+        )
+
     async def async_post_call_success_deployment_hook(
         self,
         request_data: dict,
@@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
         """
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
-        # should run guardrail
-        litellm_guardrails: Final = request_data.get("guardrails")
-        if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
-            return None
-
-        if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
+        if not self.runs_post_call_deployment_hook(request_data):
             return None
 
         target: Final = self._deployment_hook_target()

litellm/integrations/deepeval/deepeval.py

@@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
     to_zod_compatible_iso,
     validate_environment,
 )
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 
 
 # This file includes the custom callbacks for LiteLLM Proxy
@@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
             )
         else:
             output = str(standard_logging_object.get("error_string", ""))
+        logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
+        span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
         return BaseApiSpan(
-            uuid=standard_logging_object.get("id", uuid.uuid4()),
+            uuid=span_id,
             name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
             status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
             type=SpanApiType.LLM,

litellm/integrations/dynamodb.py

@@ -9,6 +9,8 @@ from typing import Final, Protocol
 
 import litellm
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 
 
 class _DynamoTable(Protocol):
@@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
 class DyanmoDBLogger:
     # Class variables or attributes
 
-    def __init__(self):
+    def __init__(self, dynamodb: _DynamoResource | None = None):
         # Instance variables
         import boto3
 
-        self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
+        self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
+            "dynamodb", region_name=os.environ["AWS_REGION_NAME"]
+        )
         if litellm.dynamodb_table_name is None:
             raise ValueError(
                 "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@@ -45,23 +49,23 @@ class DyanmoDBLogger:
 
     _async_log_event = async_log_event
 
-    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+    def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
         try:
             print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
 
             # construct payload to send to DynamoDB
             # follows the same params as langfuse.py
-            litellm_params: Final = kwargs.get("litellm_params", {})
-            metadata: Final = litellm_params.get("metadata", {}) or {}  # if litellm_params['metadata'] == None
+            litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+            metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
             messages: Final = kwargs.get("messages")
             optional_params: Final = kwargs.get("optional_params", {})
             call_type: Final = kwargs.get("call_type", "litellm.completion")
             usage: Final = response_obj["usage"]
-            id: Final = response_obj.get("id", str(uuid.uuid4()))
+            response_id: Final = response_obj.get("id", str(uuid.uuid4()))
 
             # Build the initial payload
             payload: Final[dict[str, object]] = {
-                "id": id,
+                "id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
                 "call_type": call_type,
                 "startTime": start_time,
                 "endTime": end_time,

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
     sanitize_cloud_object_component,
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
 from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         else:
             await self._send_individual_logs(items_to_process)
 
-    def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
+    def _get_object_name(
+        self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
+    ) -> str:
         """
         Get the object name to use for the current payload
         """
         current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
+        metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
+        gcs_log_id: Final = metadata.get("gcs_log_id")
+        safe_log_id: Final = sanitize_cloud_object_component(
+            gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
+        )
+        if safe_log_id:
+            return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
         if logging_payload.get("error_str", None) is not None:
-            object_name = self._generate_failure_object_name(
-                request_date_str=current_date,
-            )
-        else:
-            object_name = self._generate_success_object_name(
-                request_date_str=current_date,
-                response_id=response_obj.get("id", ""),
-            )
-
-        # used for testing
-        _litellm_params: Final = kwargs.get("litellm_params", None) or {}
-        _metadata: Final = _litellm_params.get("metadata", None) or {}
-        if "gcs_log_id" in _metadata:
-            safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
-            if safe_log_id:
-                object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
-
-        return object_name
+            return self._generate_failure_object_name(request_date_str=current_date)
+        raw_response_id: Final[object] = response_obj.get("id", "")
+        response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
+        return self._generate_success_object_name(
+            request_date_str=current_date,
+            response_id=logged_event_key(kwargs, response_id),
+        )
 
     async def get_request_response_payload(
         self,

litellm/integrations/openmeter.py

@@ -3,12 +3,15 @@
 
 import json
 import os
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
     get_async_httpx_client,
@@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
         if len(missing_keys) > 0:
             raise Exception(f"Missing keys={missing_keys} in environment.")
 
-    def _common_logic(self, kwargs: dict, response_obj):
+    def _common_logic(self, kwargs: Mapping[str, object], response_obj):
         call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
         dt: Final = get_utc_datetime().isoformat()
         cost: Final = kwargs.get("response_cost", None)
@@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
         # If no user provided directly, try to get it from token user_id
         if user_param is None:
             # Check if user_id is available from the API key metadata
-            litellm_params: Final = kwargs.get("litellm_params", {})
-            metadata: Final = litellm_params.get("metadata", {})
+            litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+            metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
             user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
 
             if user_api_key_user_id is not None:
@@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
         return {
             "specversion": "1.0",
             "type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
-            "id": call_id,
+            "id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
             "time": dt,
             "subject": subject,
             "source": "litellm-proxy",

litellm/integrations/prometheus.py

@@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
 
     async def async_post_call_failure_hook(
         self,
-        request_data: dict,
+        request_data: dict[str, object],
         original_exception: Exception,
         user_api_key_dict: UserAPIKeyAuth,
         traceback_str: str | None = None,
@@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
         try:
             _tags: Final = StandardLoggingPayloadSetup.get_request_tags(
                 litellm_params=request_data,
-                proxy_server_request=request_data.get("proxy_server_request", {}),
+                proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
+            )
+            _metadata: Final = _logged_fields(request_data.get("metadata"))
+            model_id: Final = (
+                _logged_fields(_metadata.get("model_info")).get("id")
+                or _logged_fields(request_data.get("model_info")).get("id")
+                or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
             )
-            _metadata: Final = request_data.get("metadata", {}) or {}
-            model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
             rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
             api_provider: Final = self._extract_api_provider_from_request_data(
                 request_data

litellm/integrations/websearch_interception/handler.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
 from typing_extensions import Never, ReadOnly
 
 import litellm
+from litellm._internal_context import billed_separately
 from litellm._logging import verbose_logger
 from litellm.anthropic_interface import messages as anthropic_messages
 from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
             search_kwargs: Final = MappingProxyType(
                 {**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
             )
-            result: Final = (
-                await litellm.asearch(
-                    query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+            with billed_separately():
+                result: Final = (
+                    await litellm.asearch(
+                        query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+                    )
+                    if search_metadata is None
+                    else await litellm.asearch(
+                        query=query_arg,
+                        search_provider=search_provider,
+                        litellm_metadata=search_metadata,
+                        **_NO_ASEARCH_NAMED,
+                        **search_kwargs,
+                    )
                 )
-                if search_metadata is None
-                else await litellm.asearch(
-                    query=query_arg,
-                    search_provider=search_provider,
-                    litellm_metadata=search_metadata,
-                    **_NO_ASEARCH_NAMED,
-                    **search_kwargs,
-                )
-            )
 
             # Format using transformation function
             search_result_text: Final = WebSearchTransformation.format_search_response(result)

litellm/litellm_core_utils/allowed_tools_tool_choice.py

@@ -0,0 +1,111 @@
+from collections.abc import Mapping
+from typing import Final, Literal
+
+from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+from litellm.exceptions import BadRequestError
+
+ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
+
+CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
+    '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
+)
+
+
+class ChatAllowedTools(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    mode: Literal["auto", "required"]
+    tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
+
+
+class ChatAllowedToolsToolChoice(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    type: Literal["allowed_tools"]
+    allowed_tools: ChatAllowedTools
+
+
+_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
+
+
+_SELECTOR: Final = TypeAdapter(dict[str, object])
+
+
+class _ToolChoiceType(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str
+
+
+_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType)
+
+
+def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
+    try:
+        return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools"
+    except ValidationError:
+        return False
+
+
+def raise_if_allowed_tools_tool_choice_unsupported(
+    tool_choice: object,
+    custom_llm_provider: str,
+    model: str,
+) -> None:
+    """Providers outside the set would silently send the request with no tool restriction at all, so the call stops
+    here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
+    if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
+        return
+    raise BadRequestError(
+        message=(
+            "tool_choice type 'allowed_tools' is only supported for "
+            f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
+            "drop_params does not drop it, since that would let the model call tools the request excluded. "
+            "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
+        ),
+        model=model,
+        llm_provider=custom_llm_provider,
+    )
+
+
+def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
+    try:
+        return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
+    except ValidationError:
+        return None
+
+
+def _has_name(selector: Mapping[str, object]) -> bool:
+    name: Final = selector.get("name")
+    return isinstance(name, str) and name != ""
+
+
+def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
+    tool_type: Final = tool.get("type")
+    if tool_type not in ("function", "custom"):
+        return tool
+    if tool_type not in tool:
+        return tool if _has_name(tool) else None
+    try:
+        nested: Final = _SELECTOR.validate_python(tool[tool_type])
+    except ValidationError:
+        return None
+    outer: Final = {key: value for key, value in tool.items() if key != tool_type}
+    if not _has_name(nested) or outer.keys() & nested.keys():
+        return None
+    return {**outer, **nested}
+
+
+def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None:
+    """The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is malformed or
+    carries conflicting keys, so the caller sends the original rather than a rewrite that could mean something looser."""
+    flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools)
+    if any(tool is None for tool in flattened):
+        return None
+    return ToolChoiceAllowedParam(
+        type="allowed_tools",
+        mode=choice.allowed_tools.mode,
+        tools=[dict(tool) for tool in flattened if tool is not None],
+    )

litellm/litellm_core_utils/audio_utils/transcription_streaming.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import datetime
+import json
+import traceback
+from collections.abc import AsyncIterator, Callable, Iterator, Mapping
+from typing import Final, Protocol
+
+import anyio
+from openai import AsyncStream, Stream
+from openai.types.audio import (
+    TranscriptionStreamEvent,
+    TranscriptionTextDeltaEvent,
+    TranscriptionTextDoneEvent,
+)
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    TranscriptionUsageObjectTransformation,
+)
+from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
+
+_STREAM_FLAG: Final = TypeAdapter(bool)
+
+
+def transcription_stream_flag(value: object) -> bool | None:
+    try:
+        return _STREAM_FLAG.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def transcription_stream_unset(value: object) -> bool:
+    return value is None or value == ""
+
+
+def transcription_stream_requested(value: object) -> bool:
+    return transcription_stream_flag(value) is True
+
+
+TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
+
+TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
+
+
+def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
+    from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
+
+    model: Final = call_details.get("model")
+    custom_llm_provider: Final = call_details.get("custom_llm_provider")
+    extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
+
+    def map_exception(exception: Exception) -> Exception:
+        try:
+            mapped: Final[object] = exception_type(
+                model=model if isinstance(model, str) else None,
+                original_exception=exception,
+                custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+                extra_kwargs=extra_kwargs,
+            )
+        except Exception as mapped_exception:  # noqa: BLE001  # exception_type raises the mapped exception
+            return mapped_exception
+        return mapped if isinstance(mapped, Exception) else exception
+
+    return map_exception
+
+
+def _guarded_transcription_event_payloads(
+    response: TranscriptionResponse,
+) -> tuple[dict[str, object], dict[str, object]]:
+    text: Final = response.text or ""
+    usage: Final = response.usage
+    languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
+    return (
+        {"type": "transcript.text.delta", "delta": text},
+        {
+            "type": "transcript.text.done",
+            "text": text,
+            **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
+            **({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
+        },
+    )
+
+
+def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
+    return (
+        *(
+            f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
+            for event in _guarded_transcription_event_payloads(response)
+        ),
+        TRANSCRIPTION_STREAM_DONE_FRAME,
+    )
+
+
+class GuardedTranscriptionReplay:
+    def __init__(self, response: TranscriptionResponse) -> None:
+        self.response: Final = response
+        self._hidden_params: dict[str, object] = response.hidden_params
+
+    def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
+        return self._events()
+
+    async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
+        delta, done = _guarded_transcription_event_payloads(self.response)
+        yield TranscriptionTextDeltaEvent.model_construct(**delta)
+        yield TranscriptionTextDoneEvent.model_construct(**done)
+
+
+class TranscriptionStreamLogging(Protocol):
+    def success_handler(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    async def async_success_handler(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def handle_sync_success_callbacks_for_async_calls(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def failure_handler(
+        self,
+        exception: Exception,
+        traceback_exception: str,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+        *,
+        for_async_request: bool = False,
+    ) -> None: ...
+
+    async def async_failure_handler(
+        self,
+        exception: Exception,
+        traceback_exception: str,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
+
+    def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
+
+
+class _TranscriptionEventCollector:
+    def __init__(self, duration: float | None) -> None:
+        self.duration = duration
+        self.text_deltas: list[str] = []  # mutable-ok: streaming deltas accumulate until the terminal event
+        self.done_event: TranscriptionTextDoneEvent | None = None
+
+    def add(self, event: TranscriptionStreamEvent) -> None:
+        if isinstance(event, TranscriptionTextDeltaEvent):
+            self.text_deltas.append(event.delta)
+        elif isinstance(event, TranscriptionTextDoneEvent):
+            self.done_event = event
+
+    def response(self) -> TranscriptionResponse:
+        done_event: Final = self.done_event
+        response: Final = TranscriptionResponse(
+            text=done_event.text if done_event is not None else "".join(self.text_deltas),
+            usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
+            languages=getattr(done_event, "languages", None) if done_event is not None else None,
+        )
+        if self.duration is not None:
+            response.set_audio_transcription_duration(self.duration)
+        return response
+
+
+def _billed_usage(response: TranscriptionResponse) -> Usage:
+    usage: Final[object] = getattr(response, "usage", None)
+    token_usage: Final = (
+        TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
+        if isinstance(usage, TranscriptionUsageTokensObject)
+        else None
+    )
+    return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+
+
+def _record_billable_partial_spend(
+    logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
+) -> None:
+    partial_response: Final = collector.response()
+    partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
+    if partial_cost <= 0:
+        return
+    logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
+
+
+class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
+    def __init__(
+        self,
+        stream: Stream[TranscriptionStreamEvent],
+        logging_obj: TranscriptionStreamLogging,
+        start_time: datetime.datetime,
+        map_exception: TranscriptionStreamExceptionMapper,
+    ) -> None:
+        self.__dict__.update(stream.__dict__)
+        self._hidden_params: dict[str, object] = {  # mutable-ok: ResponseMetadata fills in the call's metadata
+            "additional_headers": process_response_headers(stream.response.headers)
+        }
+        self._logging_obj = logging_obj
+        self._start_time = start_time
+        self._map_exception = map_exception
+        self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
+        self._finalized = False
+        self._failed = False
+        source_iterator: Final = self._iterator
+        self._iterator = self._logging_iterator(source_iterator)
+
+    def _logging_iterator(
+        self, source_iterator: Iterator[TranscriptionStreamEvent]
+    ) -> Iterator[TranscriptionStreamEvent]:
+        try:
+            for event in source_iterator:
+                self._collector.add(event)
+                yield event
+        except Exception as exception:
+            self._failed = True
+            _record_billable_partial_spend(self._logging_obj, self._collector)
+            mapped_exception: Final = self._map_exception(exception)
+            end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+            self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
+            raise mapped_exception from exception
+        finally:
+            self._finalize()
+
+    def _finalize(self) -> None:
+        if self._finalized or self._failed:
+            return
+        self._finalized = True
+        self._logging_obj.success_handler(
+            self._collector.response(),
+            self._start_time,
+            datetime.datetime.now(),  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+        )
+
+    def close(self) -> None:
+        try:
+            super().close()
+        finally:
+            self._finalize()
+
+    @property
+    def logging_obj(self) -> TranscriptionStreamLogging:
+        return self._logging_obj
+
+
+class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
+    def __init__(
+        self,
+        stream: AsyncStream[TranscriptionStreamEvent],
+        logging_obj: TranscriptionStreamLogging,
+        start_time: datetime.datetime,
+        map_exception: TranscriptionStreamExceptionMapper,
+    ) -> None:
+        self.__dict__.update(stream.__dict__)
+        self._hidden_params: dict[str, object] = {  # mutable-ok: ResponseMetadata fills in the call's metadata
+            "additional_headers": process_response_headers(stream.response.headers)
+        }
+        self._logging_obj = logging_obj
+        self._start_time = start_time
+        self._map_exception = map_exception
+        self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
+        self._finalized = False
+        self._failed = False
+        source_iterator: Final = self._iterator
+        self._iterator = self._logging_iterator(source_iterator)
+
+    async def _logging_iterator(
+        self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
+    ) -> AsyncIterator[TranscriptionStreamEvent]:
+        try:
+            async for event in source_iterator:
+                self._collector.add(event)
+                yield event
+        except Exception as exception:
+            self._failed = True
+            _record_billable_partial_spend(self._logging_obj, self._collector)
+            mapped_exception: Final = self._map_exception(exception)
+            end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+            self._logging_obj.failure_handler(
+                mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
+            )
+            await self._logging_obj.async_failure_handler(
+                mapped_exception,
+                traceback.format_exc(),
+                self._start_time,
+                end_time,
+            )
+            raise mapped_exception from exception
+        finally:
+            await self._finalize()
+
+    async def _finalize(self) -> None:
+        if self._finalized or self._failed:
+            return
+        self._finalized = True
+        response: Final = self._collector.response()
+        end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+        self._logging_obj.handle_sync_success_callbacks_for_async_calls(
+            result=response,
+            start_time=self._start_time,
+            end_time=end_time,
+        )
+        with anyio.CancelScope(shield=True):
+            await self._logging_obj.async_success_handler(
+                result=response,
+                start_time=self._start_time,
+                end_time=end_time,
+            )
+
+    async def close(self) -> None:
+        try:
+            await super().close()
+        finally:
+            await self._finalize()
+
+    @property
+    def logging_obj(self) -> TranscriptionStreamLogging:
+        return self._logging_obj
+
+
+def wrap_transcription_stream(
+    stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
+    logging_obj: TranscriptionStreamLogging,
+    start_time: datetime.datetime,
+    map_exception: TranscriptionStreamExceptionMapper,
+) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
+    if isinstance(stream, AsyncStream):
+        return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
+    return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)

litellm/litellm_core_utils/audio_utils/utils.py

@@ -21,6 +21,16 @@ from litellm.types.files import (
 from litellm.types.utils import FileTypes
 
 
+def normalized_audio_duration_seconds(value: object) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    try:
+        seconds: Final = float(value)
+    except OverflowError:
+        return None
+    return seconds if math.isfinite(seconds) and seconds >= 0 else None
+
+
 @dataclass
 class ProcessedAudioFile:
     """

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -8,6 +8,7 @@ from typing import Final
 
 from pydantic import TypeAdapter, ValidationError
 
+from litellm._internal_context import internal_sub_call
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@@ -204,12 +205,13 @@ async def _execute_chat_completion_agentic_plan(
     )
 
     try:
-        response_followup = await litellm.acompletion(
-            model=full_model_name,
-            messages=patch.messages,
-            **optional_params_for_followup,
-            **kwargs_for_followup,
-        )
+        with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None):
+            response_followup = await litellm.acompletion(
+                model=full_model_name,
+                messages=patch.messages,
+                **optional_params_for_followup,
+                **kwargs_for_followup,
+            )
         if _post_hook_overridden(callback):
             try:
                 response_followup = await callback.async_post_agentic_loop_response_hook(

litellm/litellm_core_utils/core_helpers.py

@@ -4,7 +4,7 @@ import copy
 import logging
 import math
 import re
-from collections.abc import Collection, Iterable, Iterator, Mapping
+from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
 
@@ -97,13 +97,17 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
     return float(seconds / denominator)
 
 
-_DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
+_LAX_BOOL: Final = TypeAdapter(bool)
 
 
 def as_str_mapping(value: object) -> Mapping[str, object] | None:
     return value if isinstance(value, Mapping) else None
 
 
+def as_object_list(value: object) -> Sequence[object] | None:
+    return value if isinstance(value, list) else None
+
+
 def decoded_json_object(value: object) -> Mapping[str, object]:
     mapping: Final = as_str_mapping(value)
     if mapping is not None:
@@ -156,15 +160,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]:
     return iter_items(value) if value else ()
 
 
-def normalize_drop_params(value: object) -> bool | None:
+def lax_bool(value: object) -> bool | None:
     if value is None or isinstance(value, bool):
         return value
     try:
-        return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
+        return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
     except ValidationError:
         return None
 
 
+def normalize_drop_params(value: object) -> bool | None:
+    return lax_bool(value)
+
+
 def drops_unsupported_params(drop_params: object) -> bool:
     return litellm.drop_params is True or normalize_drop_params(drop_params) is True
 
@@ -315,6 +323,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
+    "STOP_SEQUENCE": "stop",
+    "TOOL_CALL": "tool_calls",
     "ERROR_TOXIC": "content_filter",
     "ERROR": "stop",
     # HuggingFace / Together AI

litellm/litellm_core_utils/decisions_pricing.py

@@ -12,6 +12,7 @@ from typing import Final
 
 DECISIONS_PRICING_SUFFIX: Final = "_decisions"
 DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
+QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
 
 
 def is_decisions_call_type(call_type: str) -> bool:
@@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
     }
 
 
+def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
+    if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
+        return None
+    rates: Final = {**model_info, **overrides}
+    request_price: Final = rates.get("input_cost_per_query")
+    if not isinstance(request_price, (int, float)):
+        return None
+    bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token"))
+    return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price)
+
+
 def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
     """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
     plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -107,6 +107,7 @@ class ExceptionCheckers:
             "exceeds the available context size",  # llama.cpp/Lemonade
             "exceeds the maximum number of tokens allowed",  # Gemini
             '"code": "context_length_exceeded"',
+            "prompt is too long",
         ]
         for substring in known_exception_substrings:
             if substring in _error_str_lowercase:

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -146,6 +146,10 @@ def handle_anthropic_text_model_custom_llm_provider(
     return model, custom_llm_provider
 
 
+def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
+    return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
+
+
 def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
     """The authenticating provider this pair already names, or None.
 
@@ -154,7 +158,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
     and for a declared pair the resolver's answer is the declaration itself, so metadata callers
     adopt the declaration instead of resolving.
     """
-    declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
+    declared: Final = declared_provider(model, custom_llm_provider)
     return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
 
 

litellm/litellm_core_utils/internal_call_metadata.py

@@ -26,7 +26,12 @@ from litellm.constants import (
     MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
     NON_INFERENCE_CALL_TYPES,
 )
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
+from litellm.types.passthrough_endpoints.pass_through_endpoints import (
+    SETTLES_BACKGROUND_RESPONSE_KWARG,
+    BackgroundResponseSettlement,
+)
 from litellm.types.utils import (
     BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
     BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
@@ -35,6 +40,8 @@ from litellm.types.utils import (
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
 
+BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
+
 ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
 OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
 ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
@@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
     return is_unbilled_non_inference_call(call_type, metadata, response)
 
 
+def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
+    return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
+        call_type in NON_INFERENCE_CALL_TYPES
+        and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
+    )
+
+
+def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
+    call_type: Final = kwargs.get("call_type")
+    litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+    metadata: Final = (
+        as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
+    )
+    if not event_id or not settles_background_response(
+        call_type if isinstance(call_type, str) else "", metadata, kwargs
+    ):
+        return event_id
+    return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
+
+
 def sanitize_user_api_key_auth(auth: object) -> object:
     """Copy of the auth object with its budget reservation removed; the cost callback
     falls back to reading the reservation from inside the auth object."""

litellm/litellm_core_utils/litellm_logging.py

@@ -19,9 +19,11 @@ from datetime import datetime as dt_object
 from enum import Enum
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast
 
 from httpx import Response, Timeout
+from openai import AsyncStream as OpenAIAsyncStream
+from openai import Stream as OpenAIStream
 from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
@@ -100,6 +102,7 @@ from litellm.litellm_core_utils.hidden_params import (
     get_hidden_params,
     served_from_cache,
     set_hidden_param,
+    set_hidden_params,
 )
 from litellm.litellm_core_utils.internal_call_metadata import (
     MODEL_ACCESS_GROUP_METADATA_KEY,
@@ -125,6 +128,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
     diagnose_zero_cost,
     zero_cost_warning,
 )
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler
 from litellm.litellm_core_utils.logging_utils import (
     truncate_base64_in_messages,
     truncate_base64_in_messages_async,
@@ -145,7 +149,7 @@ from litellm.litellm_core_utils.served_output_texts import (
 from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.llms.base_llm.ocr.transformation import OCRResponse
 from litellm.llms.base_llm.search.transformation import SearchResponse
-from litellm.responses.utils import ResponseAPILoggingUtils
+from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
 from litellm.types.agents import LiteLLMSendMessageResponse
 from litellm.types.containers.main import ContainerObject
 from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
@@ -309,6 +313,11 @@ class _EnterpriseAlertingLoggers(NamedTuple):
     smtp_email: type[CustomLogger]
 
 
+class BridgedChatRequest(NamedTuple):
+    optional_params: Mapping[str, object]
+    litellm_params: Mapping[str, object]
+
+
 @functools.cache
 def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers:
     try:
@@ -589,6 +598,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
 _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
 _GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
 _REDACTED_RESULT: Final = TypeAdapter(object)
+_LoggedResponse: Final = TypeVar("_LoggedResponse")
 _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
 
 
@@ -881,6 +891,8 @@ class Logging(LiteLLMLoggingBaseClass):
         # The model the proxy shows the client on streamed chunks. The logged streamed response carries it
         # once that response is priced, the same way a non-streamed response is logged
         self.client_facing_stream_model: str | None = None
+        self.client_facing_response_id: str | None = None
+        self.bridged_chat_request: BridgedChatRequest | None = None
         self.zero_cost_warned: bool = False
         self.llm_caching_handler: LLMCachingHandler | None = None
 
@@ -927,6 +939,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # post_call guardrails have run; the @client decorator then stores the
         # enqueue closure here instead of firing it immediately.
         self.defer_async_logging: bool = False
+        self.drops_stream_token_ids: bool = False
         self.enqueue_deferred_logging: Callable[[], None] | None = None
         self._async_success_scheduled: bool = False
         self._async_success_enqueued: bool = False
@@ -2137,6 +2150,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if resolved_cache_hit is True:
             return 0.0
 
+        if isinstance(result, (OpenAIStream, OpenAIAsyncStream)):
+            return None
+
         if is_unbilled_non_inference_call(
             self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
         ):
@@ -3276,6 +3292,40 @@ class Logging(LiteLLMLoggingBaseClass):
             return response
         return response.model_copy(update={"model": model})
 
+    def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse:
+        request: Final = self.bridged_chat_request
+        if request is None or not isinstance(response, ResponsesAPIResponse):
+            return response
+        return self._bridged_chat_response(response, request)
+
+    def _as_served_response(
+        self, response: _LoggedResponse
+    ) -> _LoggedResponse | ModelResponse | TextCompletionResponse:
+        request: Final = self.bridged_chat_request
+        if request is None or not isinstance(response, ResponsesAPIResponse):
+            return response
+        served: Final = self._bridged_chat_response(response, request)
+        if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value):  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
+            return served
+        return self._as_text_completion_priced_as_chat(served)
+
+    def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse:
+        text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion(
+            served, TextCompletionResponse()
+        )
+        set_hidden_params(text_completion, dict(get_hidden_params(served) or {}))
+        return text_completion
+
+    def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse:
+        translated: Final = self._translate_responses_api_response_to_model_response(response, request)
+        served: Final = translated.model_copy(
+            update={
+                "id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id)
+            }
+        )
+        set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})})
+        return served
+
     def _with_continued_response_id(
         self,
         response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
@@ -3285,6 +3335,23 @@ class Logging(LiteLLMLoggingBaseClass):
             return response
         return response.model_copy(update={"id": response_id})
 
+    def _with_client_facing_response_id(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        response_id: Final = self.client_facing_response_id
+        if response_id is None or getattr(response, "id", None) == response_id:
+            return response
+        return response.model_copy(update={"id": response_id})
+
+    def _client_facing_streaming_response(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        return self._with_client_facing_response_id(
+            self._with_continued_response_id(self._with_client_facing_stream_model(response))
+        )
+
     def _success_handler_helper_fn(
         self,
         result: object = None,
@@ -3320,7 +3387,14 @@ class Logging(LiteLLMLoggingBaseClass):
                 start_time = self.start_time
             if end_time is None:
                 end_time = datetime.datetime.now()
-            if self.completion_start_time is None:
+            # A request re-dispatched to a fallback reuses this logging object but runs in
+            # its own window, so a first-token time from the previous attempt cannot belong
+            # to it. Keeping it makes the attempt report a negative TTFT.
+            first_token_seconds: Final = _timestamp_seconds(self.completion_start_time)
+            attempt_start_seconds: Final = _timestamp_seconds(start_time)
+            if first_token_seconds is None or (
+                attempt_start_seconds is not None and first_token_seconds < attempt_start_seconds
+            ):
                 self.completion_start_time = end_time
                 self.model_call_details["completion_start_time"] = self.completion_start_time
 
@@ -3333,6 +3407,8 @@ class Logging(LiteLLMLoggingBaseClass):
             )
 
             result = self.normalize_endpoint_logging_result(result)
+            if self.stream is not True:
+                result = self._as_served_response(result)  # rebind-ok: bridged chat calls log the served response
 
             logging_result: Final = self.normalize_logging_result(result=result)
             if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@@ -3647,6 +3723,7 @@ class Logging(LiteLLMLoggingBaseClass):
         verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
         if not self.should_run_logging(event_type="sync_success"):  # prevent double logging
             return
+        provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
         start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
             start_time=start_time,
             end_time=end_time,
@@ -3661,7 +3738,12 @@ class Logging(LiteLLMLoggingBaseClass):
         try:
             ## BUILD COMPLETE STREAMED RESPONSE
             complete_streaming_response: (
-                ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
+                ModelResponse
+                | TextCompletionResponse
+                | ResponsesAPIResponse
+                | InteractionsAPIResponse
+                | TranscriptionResponse
+                | None
             ) = None
             if "complete_streaming_response" in self.model_call_details and not for_async_request:
                 return  # break out of this.
@@ -3677,19 +3759,21 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["complete_streaming_response"] = complete_streaming_response
                 self._surface_response_headers_from_result(complete_streaming_response)
                 self.model_call_details["response_cost"] = self.response_cost_calculator(
-                    result=complete_streaming_response
+                    result=complete_streaming_response if provider_transcription is None else provider_transcription
                 )
                 self._add_continued_partial_spend(complete_streaming_response)
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
-                logged_streaming_response: Final = self._with_continued_response_id(
-                    self._with_client_facing_stream_model(complete_streaming_response)
+                logged_streaming_response: Final = self._client_facing_streaming_response(
+                    self._as_served_chat_response(complete_streaming_response)
                 )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
                 self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
                 ## STANDARDIZED LOGGING PAYLOAD
                 if not for_async_request:
                     self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
-                        logged_streaming_response, start_time, end_time
+                        logged_streaming_response if provider_transcription is None else provider_transcription,
+                        start_time,
+                        end_time,
                     )
                     standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
                         "standard_logging_object"
@@ -4216,6 +4300,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
             )
         )
+        provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
         start_time, end_time, result = self._success_handler_helper_fn(
             start_time=start_time,
             end_time=end_time,
@@ -4235,9 +4320,7 @@ class Logging(LiteLLMLoggingBaseClass):
         ## BUILD COMPLETE STREAMED RESPONSE
         if "async_complete_streaming_response" in self.model_call_details:
             return  # break out of this.
-        complete_streaming_response: Final[
-            ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
-        ] = self._get_assembled_streaming_response(
+        complete_streaming_response: Final = self._get_assembled_streaming_response(
             result=result,
             start_time=start_time,
             end_time=end_time,
@@ -4259,7 +4342,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     get_base_model_from_metadata(model_call_details=self.model_call_details)
                     # base_model defaults to None if not set on model_info
                     self.model_call_details["response_cost"] = self.response_cost_calculator(
-                        result=complete_streaming_response
+                        result=complete_streaming_response if provider_transcription is None else provider_transcription
                     )
 
                 verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"])
@@ -4280,15 +4363,17 @@ class Logging(LiteLLMLoggingBaseClass):
 
             await self._prepare_baseline_cache_estimate(complete_streaming_response)
 
-            logged_streaming_response: Final = self._with_continued_response_id(
-                self._with_client_facing_stream_model(complete_streaming_response)
+            logged_streaming_response: Final = self._client_facing_streaming_response(
+                self._as_served_chat_response(complete_streaming_response)
             )
             self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
 
             ## STANDARDIZED LOGGING PAYLOAD
             try:
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
-                    logged_streaming_response, start_time, end_time
+                    logged_streaming_response if provider_transcription is None else provider_transcription,
+                    start_time,
+                    end_time,
                 )
             except Exception:  # noqa: BLE001  # payload build must never block later callbacks (slot release)
                 verbose_logger.exception(
@@ -5158,10 +5243,17 @@ class Logging(LiteLLMLoggingBaseClass):
         end_time: datetime.datetime,
         is_async: bool,
         streaming_chunks: list[object],
-    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
+    ) -> (
+        ModelResponse
+        | TextCompletionResponse
+        | ResponsesAPIResponse
+        | InteractionsAPIResponse
+        | TranscriptionResponse
+        | None
+    ):
         if self.stream is not True:
             return None
-        if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
+        if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)):
             return result
         elif isinstance(
             result,
@@ -5232,6 +5324,13 @@ class Logging(LiteLLMLoggingBaseClass):
         row is built from this response rather than re-parsed from the served body."""
         self.bridged_model_response = response
 
+    def record_bridged_chat_request(
+        self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
+    ) -> None:
+        self.bridged_chat_request = BridgedChatRequest(
+            MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params))
+        )
+
     def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
         self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
 
@@ -5303,6 +5402,9 @@ class Logging(LiteLLMLoggingBaseClass):
         ):
             result = result.response
         if isinstance(result, ResponsesAPIResponse):
+            bridged_request: Final = self.bridged_chat_request
+            if bridged_request is not None:
+                return self._bridged_chat_response(result, bridged_request)
             return self._translate_responses_api_response_to_model_response(result)
 
         raw_body: Final[object] = result
@@ -5354,7 +5456,9 @@ class Logging(LiteLLMLoggingBaseClass):
         set_response_cost_in_hidden_params(result, reported_cost)
         return result
 
-    def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse:
+    def _translate_responses_api_response_to_model_response(
+        self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None
+    ) -> ModelResponse:
         """
         Convert a Responses API response into a ModelResponse for spend_logs.
 
@@ -5377,8 +5481,8 @@ class Logging(LiteLLMLoggingBaseClass):
                 logging_obj=self,
                 request_data={},
                 messages=[],
-                optional_params={},
-                litellm_params={},
+                optional_params={} if request is None else dict(request.optional_params),
+                litellm_params={} if request is None else dict(request.litellm_params),
                 encoding=litellm.encoding,
             )
             translated.id = served_id or translated.id

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost(
 class PromptTokensDetailsResult(TypedDict):
     cache_hit_tokens: int
     cache_hit_audio_tokens: ReadOnly[int]
+
+    cached_text_tokens: ReadOnly[int]
+    cached_audio_tokens: ReadOnly[int]
+    cached_image_tokens: ReadOnly[int]
+    has_cached_tokens_details: ReadOnly[bool]
     cache_creation_tokens: int
     cache_creation_token_details: CacheCreationTokenDetails | None
     text_tokens: int
@@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
     return PromptTokensDetailsResult(
         cache_hit_tokens=cache_hit_tokens,
         cache_hit_audio_tokens=cached_audio_tokens,
+        cached_text_tokens=cached_text_tokens,
+        cached_audio_tokens=cached_audio_tokens,
+        cached_image_tokens=cached_image_tokens,
+        has_cached_tokens_details=cached_tokens_details is not None,
         cache_creation_tokens=cache_creation_tokens,
         cache_creation_token_details=cache_creation_token_details,
         text_tokens=text_tokens,
@@ -1234,15 +1243,11 @@ def _calculate_input_cost(
     prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
 
     ### CACHE READ COST - Now uses tiered pricing
-    cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
-    audio_cache_read_rate: Final = get_cost_per_unit(
-        model_info,
-        get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
-        None,
-    )
-    prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
-    prompt_cost += float(cache_hit_audio_tokens) * (
-        audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
+    prompt_cost += _calculate_cache_read_cost(
+        prompt_tokens_details=prompt_tokens_details,
+        model_info=model_info,
+        cache_read_cost=cache_read_cost,
+        service_tier=service_tier,
     )
 
     ### AUDIO COST
@@ -1331,6 +1336,38 @@ def _calculate_input_cost(
     return prompt_cost
 
 
+def _calculate_cache_read_cost(
+    prompt_tokens_details: PromptTokensDetailsResult,
+    model_info: ModelInfo,
+    cache_read_cost: float,
+    service_tier: str | None,
+) -> float:
+    cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"]
+    cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"]
+    cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"]
+    classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
+    unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0)
+    total_cost = (  # rebind-ok: cached modality components accumulate into one cache-read cost
+        float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost
+    )
+
+    if cached_audio_tokens:
+        cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier)
+        cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost)
+        total_cost += (  # rebind-ok: cached audio contributes to cache-read cost
+            float(cached_audio_tokens) * float(cached_audio_cost or 0.0)
+        )
+
+    if cached_image_tokens:
+        cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier)
+        cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost)
+        total_cost += (  # rebind-ok: cached images contribute to cache-read cost
+            float(cached_image_tokens) * float(cached_image_cost or 0.0)
+        )
+
+    return total_cost
+
+
 def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
     """
     Resolve the per-model regional-processing uplift multiplier for a given
@@ -1502,6 +1539,10 @@ def generic_cost_per_token(
     prompt_tokens_details = PromptTokensDetailsResult(
         cache_hit_tokens=0,
         cache_hit_audio_tokens=0,
+        cached_text_tokens=0,
+        cached_audio_tokens=0,
+        cached_image_tokens=0,
+        has_cached_tokens_details=False,
         cache_creation_tokens=0,
         cache_creation_token_details=None,
         text_tokens=usage.prompt_tokens,
@@ -1680,6 +1721,7 @@ class BilledTokenRates:
     cache_creation_input_token_cost: float
     cache_creation_input_token_cost_by_ttl: Mapping[str, float]
     output_cost_per_reasoning_token: float
+    cache_read_input_image_token_cost: float | None = None
 
     def scaled(self, multiplier: float) -> "BilledTokenRates":
         if multiplier == 1.0:
@@ -1694,6 +1736,11 @@ class BilledTokenRates:
                 {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
             ),
             output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
+            cache_read_input_image_token_cost=(
+                self.cache_read_input_image_token_cost * multiplier
+                if self.cache_read_input_image_token_cost is not None
+                else None
+            ),
         )
 
 
@@ -1802,6 +1849,11 @@ def _cost_map_billed_rates(
         cache_creation_input_token_cost=base_costs.cache_creation_rate,
         cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
         output_cost_per_reasoning_token=reasoning_rate,
+        cache_read_input_image_token_cost=_get_cost_per_unit(
+            model_info,
+            _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier),
+            None,
+        ),
     ).scaled(multiplier)
 
 
@@ -1882,6 +1934,12 @@ def get_token_type_cost_breakdown(
     cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
         usage
     )
+    cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"]
+    image_cache_read_rate: Final = (
+        rates.cache_read_input_image_token_cost
+        if rates.cache_read_input_image_token_cost is not None
+        else rates.cache_read_input_token_cost
+    )
     cache_creation_cost: Final = (
         float(cache_creation_tokens) * rates.cache_creation_input_token_cost
         if custom_cost_per_token is not None
@@ -1895,8 +1953,9 @@ def get_token_type_cost_breakdown(
     return TokenTypeCostBreakdown(
         reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
         cache_read_cost=(
-            float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
+            float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost
             + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
+            + float(cached_image_tokens) * image_cache_read_rate
         ),
         cache_creation_cost=cache_creation_cost,
         rates=rates,

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -3,7 +3,8 @@ import json
 import re
 import time
 import traceback
-from collections.abc import Generator, Iterator, Mapping, Sequence
+from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
+from types import MappingProxyType
 from typing import Final, Literal, cast
 
 from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@@ -11,7 +12,12 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
+from litellm.constants import (
+    RESPONSE_FORMAT_TOOL_NAME,
+    STREAM_CHUNK_CHOICE_FIELDS,
+    STREAM_FINISH_CHOICE_FIELDS,
+    VLLM_CHOICE_FIELDS,
+)
 from litellm.litellm_core_utils.hidden_params import get_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_reasoning_content,
@@ -170,6 +176,28 @@ def _split_assembled_content_for_replay(content: str | None) -> list[str]:
     return _REPLAY_CONTENT_SLICE_RE.findall(content)
 
 
+def _drop_choice_fields(choice: StreamingChoices, fields: Iterable[str]) -> None:
+    for field in fields:
+        if hasattr(choice, field):
+            delattr(choice, field)
+
+
+def _vllm_choice_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
+    cached_fields: Final = _choice_provider_specific_fields(choice)
+    return MappingProxyType(
+        {
+            field: choice[field] if field in choice else cached_fields[field]
+            for field in VLLM_CHOICE_FIELDS
+            if field in choice or field in cached_fields
+        }
+    )
+
+
+def _choice_provider_specific_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
+    fields: Final = choice.get("provider_specific_fields")
+    return _FIELD_MAP.validate_python(fields) if isinstance(fields, dict) else MappingProxyType({})
+
+
 _REPLAY_REASONING_FIELDS: Final = frozenset({"reasoning_content", "thinking_blocks", "reasoning_items"})
 _REPLAY_LEAD_FIELDS: Final = _REPLAY_REASONING_FIELDS | {"role", "provider_specific_fields"}
 _REPLAY_TOOL_FIELDS: Final = frozenset({"tool_calls", "function_call"})
@@ -220,8 +248,9 @@ def _replay_frame(
     choice.finish_reason = finish_reason
     if not keeps_choice_metadata:
         choice.logprobs = None
-        if hasattr(choice, "enhancements"):
-            del choice.enhancements
+        _drop_choice_fields(choice, ("enhancements", *STREAM_CHUNK_CHOICE_FIELDS))
+    if finish_reason is None:
+        _drop_choice_fields(choice, STREAM_FINISH_CHOICE_FIELDS)
     return frame
 
 
@@ -325,7 +354,9 @@ async def convert_to_streaming_response_async(
 
         logprobs = choice.get("logprobs", None)
 
-        choice = StreamingChoices(finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs)
+        choice = StreamingChoices(
+            finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs, **_vllm_choice_fields(choice)
+        )
         choice_list.append(choice)
 
     model_response_object.choices = choice_list
@@ -388,6 +419,7 @@ def convert_to_streaming_response(
             delta=delta,
             logprobs=logprobs,
             enhancements=enhancements,
+            **_vllm_choice_fields(choice),
         )
 
         choice_list.append(choice)
@@ -872,6 +904,7 @@ def convert_to_model_response_object(
                     logprobs=logprobs,
                     enhancements=enhancements,
                     provider_specific_fields=provider_specific_fields,
+                    **_vllm_choice_fields(choice),
                 )
                 if provider_sent_logprobs and logprobs is None:
                     choice.logprobs = None
@@ -937,9 +970,11 @@ def convert_to_model_response_object(
             model_response_object.data = response_object["data"]
 
             if "usage" in response_object and response_object["usage"] is not None:
-                model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
-                model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
-                model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
+                embedding_usage: Final = model_response_object.usage or Usage()
+                embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
+                embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
+                embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0)
+                model_response_object.usage = embedding_usage
 
             if start_time is not None and end_time is not None:
                 model_response_object._response_ms = (
@@ -982,6 +1017,10 @@ def convert_to_model_response_object(
                 if key in response_object:
                     setattr(model_response_object, key, response_object[key])
 
+            detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages"))
+            if detected_languages is not None:
+                model_response_object.languages = detected_languages
+
             if "usage" in response_object and response_object["usage"] is not None:
                 tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None
 

litellm/litellm_core_utils/model_param_helper.py

@@ -1,6 +1,6 @@
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import ClassVar, Final
+from typing import ClassVar, Final, get_args
 
 from openai.types.chat.completion_create_params import (
     CompletionCreateParamsNonStreaming,
@@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.anthropic import AnthropicMessagesRequest
+from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
 from litellm.types.rerank import RerankRequest
 
 
@@ -178,6 +179,7 @@ class ModelParamHelper:
 
         This follows the OpenAI API Spec
         """
+        litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams))
         try:
             from openai.types.audio.transcription_create_params import (
                 TranscriptionCreateParamsNonStreaming,
@@ -187,11 +189,11 @@ class ModelParamHelper:
             non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys())
             streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys())
 
-            all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs)
+            all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs)
             return all_transcription_kwargs
         except Exception as e:
             verbose_logger.debug("Error getting transcription kwargs %s", str(e))
-            return set()
+            return litellm_transcription_kwargs
 
     @staticmethod
     def _get_litellm_supported_responses_api_kwargs() -> set[str]:

litellm/litellm_core_utils/pattern_match_deployments.py

@@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching
 import copy
 import itertools
 import re
+from collections.abc import Mapping
 from re import Match
 from typing import Final
 
@@ -22,6 +23,16 @@ class _RoutedDeployment(BaseModel):
 
 
 _ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
+_PATTERN_WIDE_CACHE_FLAGS: Final = frozenset(("supports_prompt_cache_breakpoint", "supports_prompt_caching"))
+_AZURE_AI_PATTERN_CACHE_FLAGS: Final = frozenset(("supports_prompt_caching",))
+
+
+def _cache_flags_left_to_the_catalog(litellm_params: Mapping[str, object]) -> frozenset[str]:
+    model: Final = litellm_params.get("model")
+    provider: Final = litellm_params.get("custom_llm_provider") or inferred_provider(
+        model if isinstance(model, str) else None
+    )
+    return _AZURE_AI_PATTERN_CACHE_FLAGS if provider == "azure_ai" else _PATTERN_WIDE_CACHE_FLAGS
 
 
 class PatternUtils:
@@ -133,6 +144,11 @@ class PatternMatchRouter:
                 matched_pattern=matched_pattern,
                 litellm_deployment_litellm_model=deployment["litellm_params"]["model"],
             )
+            if "*" in deployment["litellm_params"]["model"] and isinstance(new_deployment.get("model_info"), dict):
+                left_to_the_catalog = _cache_flags_left_to_the_catalog(deployment["litellm_params"])
+                new_deployment["model_info"] = {
+                    key: value for key, value in new_deployment["model_info"].items() if key not in left_to_the_catalog
+                }
             new_deployments.append(new_deployment)
 
         return new_deployments

litellm/litellm_core_utils/prompt_templates/chat_template_tools.py

@@ -10,12 +10,18 @@ class ChatTemplateToolSection:
     preamble_tokens: int
     implicit_system_turn_tokens: int
     render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
+    counts_tool_choice: bool = False
+    renders_strict_only_when_true: bool = False
 
 
-def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
     return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
 
 
+def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions)
+
+
 def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
     return "".join(
         "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
@@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
 
 
 _TOOL_SECTIONS: Final = (
+    (
+        re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE),
+        ChatTemplateToolSection(211, 0, _json_line_tool_schemas),
+    ),
     (
         re.compile(
             r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
         ),
-        ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
+        ChatTemplateToolSection(222, 0, _json_line_tool_schemas),
     ),
     (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
+    (
+        re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
+        ChatTemplateToolSection(
+            0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
+        ),
+    ),
+    (
+        re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
+        ChatTemplateToolSection(
+            0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
+        ),
+    ),
 )
 
 

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -8,7 +8,7 @@ import io
 import json
 import mimetypes
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from graphlib import CycleError, TopologicalSorter
 from itertools import chain, groupby, islice
@@ -21,14 +21,17 @@ from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.batch_utils import InMemoryFile
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionAssistantMessage,
     ChatCompletionAudioObject,
+    ChatCompletionCachedContent,
     ChatCompletionFileObject,
     ChatCompletionImageObject,
+    ChatCompletionPromptCacheBreakpoint,
     ChatCompletionReasoningItem,
     ChatCompletionReasoningSummaryTextBlock,
     ChatCompletionRedactedThinkingBlock,
@@ -2182,6 +2185,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
 
 
 _MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
+PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = (
+    "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)"
+)
 
 
 def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
@@ -2193,16 +2199,62 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
 
 
 _PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
+_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint)
+_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object])
+
+
+def _accepted_prompt_cache_breakpoint(
+    accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
+) -> _AcceptedBreakpointT | None:
+    try:
+        return accepted.validate_python(marker)
+    except ValidationError:
+        return None
+
+
+def prompt_cache_breakpoint_within(
+    accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
+) -> _AcceptedBreakpointT | None:
+    """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept,
+    and a marker whose mode the target cannot read is dropped whole."""
+    whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker)
+    if whole is not None:
+        return whole
+    fields: Final = _breakpoint_fields_or_none(marker)
+    without_ttl: Final = (
+        None
+        if fields is None
+        else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"})
+    )
+    verbose_logger.debug(
+        "Dropping malformed prompt_cache_breakpoint %r under drop_params"
+        if without_ttl is None
+        else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params",
+        marker,
+    )
+    return without_ttl
 
 
 def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
     if marker is None or not drop_params:
         return marker
-    try:
-        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
-    except ValidationError:
-        verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
-        return None
+    return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker)
+
+
+def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object:
+    """The client's marker as written, or None under drop_params when no target could read its mode, so the
+    target's own config decides which ttl and keys it keeps."""
+    return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker
+
+
+def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None:
+    return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker)
+
+
+def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
+    if marker is None or not drop_params:
+        return marker
+    return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker)
 
 
 def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
@@ -2215,11 +2267,63 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
     )
 
 
+def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT:
+    """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block.
+    A ``cache_control`` the client already wrote on the block wins, TTL included."""
+    marker: Final = block.get("prompt_cache_breakpoint")
+    if marker is None:
+        return block
+    unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
+    control: Final = (
+        None
+        if block.get("cache_control") is not None
+        else _cache_control_for_prompt_cache_breakpoint(marker, drop_params)
+    )
+    marked: Final = unmarked if control is None else {**unmarked, "cache_control": control}
+    return cast(_MarkedT, marked)  # cast-ok: same block shape as the input with the marker key renamed
+
+
+def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None:
+    try:
+        breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
+    except ValidationError:
+        if not drop_params:
+            raise UnsupportedParamsError(
+                message=(
+                    f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got "
+                    f"{marker!r}. Fix the marker, or "
+                    f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
+                ),
+                status_code=400,
+            )
+        verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
+        return None
+    ttl: Final = breakpoint_mark.get("ttl")
+    if ttl is not None and not drop_params:
+        raise UnsupportedParamsError(
+            message=(
+                f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint "
+                f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send "
+                "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block "
+                "where the model offers a 1h cache."
+            ),
+            status_code=400,
+        )
+    return ChatCompletionCachedContent(type="ephemeral")
+
+
 _BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
 _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
 _BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
 
 
+def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None:
+    try:
+        return _BREAKPOINT_FIELDS.validate_python(value)
+    except ValidationError:
+        return None
+
+
 def _without_prompt_cache_breakpoint(value: object) -> object:
     if not isinstance(value, Mapping):
         return value
@@ -2242,6 +2346,27 @@ def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
     return cast(_MarkedT, stripped)  # cast-ok: same block shape as the input minus the marker keys
 
 
+def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object:
+    if not isinstance(part, Mapping):
+        return part
+    fields: Final = _BREAKPOINT_FIELDS.validate_python(part)
+    marker: Final = fields.get("prompt_cache_breakpoint")
+    if marker is None:
+        return fields
+    unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
+    return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True))
+
+
+def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT:
+    content: Final = message.get("content")
+    if not drop_params or not isinstance(content, list):
+        return message
+    parts: Final = [
+        _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content)
+    ]
+    return cast(_MarkedT, {**message, "content": parts})  # cast-ok: same message shape with its parts' markers trimmed
+
+
 LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset(
     {"thinking_blocks", "reasoning_items", "provider_specific_fields"}
 )
@@ -3235,3 +3360,15 @@ def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues
     if string_entries and len(string_entries) == len(entries):
         return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in string_entries)
     raise ValueError(f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {prompt_type_name}.")
+
+
+def is_present_non_string(value: object) -> bool:
+    return value is not None and not isinstance(value, str)
+
+
+def invalid_type_message(field_path: str, value: object) -> str:
+    return f"Invalid type for '{field_path}': expected a string, got {type(value).__name__}"
+
+
+def is_unhashable_value_behind(error: TypeError, value: object) -> bool:
+    return not isinstance(value, Hashable) and f"unhashable type: '{type(value).__name__}'" in str(error)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -32,7 +32,9 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
+    string_field,
 )
+from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
 from litellm.types.files import get_file_extension_from_mime_type
@@ -75,7 +77,7 @@ from .common_utils import (
     parse_tool_call_arguments,
     salvage_concatenated_tool_arguments,
 )
-from .image_handling import convert_url_to_base64
+from .image_handling import convert_url_to_base64, is_pdf_media_type
 
 
 def default_pt(messages):
@@ -909,8 +911,17 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) ->
         )
 
 
+def _anthropic_https_url_block(
+    image_url: str, image_format: str | None
+) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
+    source: Final = AnthropicContentParamSourceUrl(type="url", url=image_url)
+    if image_format and is_pdf_media_type(image_format):
+        return AnthropicMessagesDocumentParam(type="document", source=source)
+    return AnthropicMessagesImageParam(type="image", source=source)
+
+
 def create_anthropic_image_param(
-    image_url_input: str | dict,
+    image_url_input: str | Mapping[str, object],
     format: str | None = None,
     is_bedrock_invoke: bool = False,
 ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
@@ -919,13 +930,11 @@ def create_anthropic_image_param(
 
     Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
     """
-    # Extract URL and format from input
-    if isinstance(image_url_input, str):
-        image_url = image_url_input
-    else:
-        image_url = image_url_input.get("url", "")
-        if format is None:
-            format = image_url_input.get("format")
+    url_value: Final = image_url_input if isinstance(image_url_input, str) else image_url_input.get("url")
+    format_value: Final = None if isinstance(image_url_input, str) else image_url_input.get("format")
+    image_url: Final = url_value if isinstance(url_value, str) else ""
+    explicit_format: Final = format if format is not None else format_value
+    image_format: Final = explicit_format if isinstance(explicit_format, str) else None
 
     # Check if the image URL is an HTTP/HTTPS URL
     if image_url.startswith("http://") or image_url.startswith("https://"):
@@ -933,21 +942,14 @@ def create_anthropic_image_param(
         # as these providers don't support URL sources for images
         if is_bedrock_invoke or image_url.startswith("http://"):
             base64_url: Final = convert_url_to_base64(url=image_url)
-            image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
-            return _anthropic_content_element_factory(image_chunk)
+            image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=image_format)
+            return anthropic_content_element_factory(image_chunk)
         else:
-            # HTTPS URL - pass directly for regular Anthropic
-            return AnthropicMessagesImageParam(
-                type="image",
-                source=AnthropicContentParamSourceUrl(
-                    type="url",
-                    url=image_url,
-                ),
-            )
+            return _anthropic_https_url_block(image_url, image_format)
     else:
         # Convert to base64 for data URIs or other formats
-        image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
-        return _anthropic_content_element_factory(image_chunk)
+        image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=image_format)
+        return anthropic_content_element_factory(image_chunk)
 
 
 # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@@ -1327,7 +1329,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
 _get_thought_signature_from_tool = get_thought_signature_from_tool
 
 
-def _get_dummy_thought_signature() -> str:
+def get_dummy_thought_signature() -> str:
     """Generate a dummy thought signature for models that require it.
 
     This is used when transferring conversation history from older models
@@ -1414,7 +1416,7 @@ def convert_to_gemini_tool_call_invoke(
                         # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
                         is_first_function_call = len(_parts_list) == 0
                         if not thought_signature and is_first_function_call and needs_dummy_signature:
-                            thought_signature = _get_dummy_thought_signature()
+                            thought_signature = get_dummy_thought_signature()
                         if thought_signature:
                             part_dict["thoughtSignature"] = thought_signature
 
@@ -1438,7 +1440,7 @@ def convert_to_gemini_tool_call_invoke(
 
                 # If no signature found and model is gemini-3, use dummy signature
                 if not thought_signature and needs_dummy_signature:
-                    thought_signature = _get_dummy_thought_signature()
+                    thought_signature = get_dummy_thought_signature()
 
                 if thought_signature:
                     part_dict_function["thoughtSignature"] = thought_signature
@@ -2023,7 +2025,7 @@ def _server_tool_answers_replaced_after(messages: Sequence[object], index: int)
     following: Final = _messages_until_the_next_assistant(messages, index)
     a_user_turn_remains: Final = any(
         message_field(message, "role") in ("user", "tool", "function")
-        and message_field(message, "tool_call_id") not in kept_server_result_ids
+        and string_field(message, "tool_call_id") not in kept_server_result_ids
         for message in following
     )
     return _tool_call_ids_in_tool_messages(following) & kept_server_result_ids if a_user_turn_remains else frozenset()
@@ -2253,16 +2255,16 @@ def add_cache_control_to_content(
     return anthropic_content_element
 
 
-def _anthropic_content_element_factory(
+def anthropic_content_element_factory(
     image_chunk: GenericImageParsingChunk,
 ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
-    if image_chunk["media_type"] == "application/pdf":
+    if is_pdf_media_type(image_chunk["media_type"]):
         _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = (
             AnthropicMessagesDocumentParam(
                 type="document",
                 source=AnthropicContentParamSource(
                     type="base64",
-                    media_type=image_chunk["media_type"],
+                    media_type="application/pdf",
                     data=image_chunk["data"],
                 ),
             )
@@ -4252,6 +4254,7 @@ def _convert_to_bedrock_tool_call_invoke(
     tool_calls: list,
     model: str | None = None,
     deployment_model: str | None = None,
+    model_info: Mapping[str, object] | None = None,
 ) -> list[BedrockContentBlock]:
     """
     OpenAI tool invokes:
@@ -4336,6 +4339,7 @@ def _convert_to_bedrock_tool_call_invoke(
                                     block_type="content_block",
                                     model=model,
                                     deployment_model=deployment_model,
+                                    model_info=model_info,
                                 )
                                 if _cache_point_block is not None:
                                     _parts_list.append(_cache_point_block)
@@ -4356,6 +4360,7 @@ def _convert_to_bedrock_tool_call_invoke(
                         block_type="content_block",
                         model=model,
                         deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     if cache_point_block is not None:
                         _parts_list.append(cache_point_block)
@@ -5027,6 +5032,7 @@ class BedrockConverseMessagesProcessor:
         deployment_model: str | None = None,
         base_model: str | None = None,
         drop_params: bool = False,
+        model_info: Mapping[str, object] | None = None,
     ) -> list[BedrockMessageBlock]:
         from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking
 
@@ -5089,6 +5095,7 @@ class BedrockConverseMessagesProcessor:
                                 block_type="content_block",
                                 model=model,
                                 deployment_model=deployment_model,
+                                model_info=model_info,
                             )
                             if _cache_point_block is not None:
                                 _parts.append(_cache_point_block)
@@ -5096,7 +5103,11 @@ class BedrockConverseMessagesProcessor:
                 elif message_content and isinstance(message_content, str):
                     _part = BedrockContentBlock(text=message_content)
                     _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
-                        message_block, block_type="content_block", model=model, deployment_model=deployment_model
+                        message_block,
+                        block_type="content_block",
+                        model=model,
+                        deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     user_content.append(_part)
                     if _cache_point_block is not None:
@@ -5147,6 +5158,7 @@ class BedrockConverseMessagesProcessor:
                         block_type="content_block",
                         model=model,
                         deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     if cache_point_block is not None:
                         tool_content.append(cache_point_block)
@@ -5212,6 +5224,7 @@ class BedrockConverseMessagesProcessor:
                             block_type="content_block",
                             model=model,
                             deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if _cache_point_block is not None and not (
                             drop_params and _as_thinking_block(element) is not None
@@ -5229,6 +5242,7 @@ class BedrockConverseMessagesProcessor:
                         block_type="content_block",
                         model=model,
                         deployment_model=deployment_model,
+                        model_info=model_info,
                     )
                     if _cache_point_block is not None:
                         assistant_content.append(_cache_point_block)
@@ -5237,7 +5251,7 @@ class BedrockConverseMessagesProcessor:
                 if _tool_calls:
                     assistant_content.extend(
                         _convert_to_bedrock_tool_call_invoke(
-                            _tool_calls, model=model, deployment_model=deployment_model
+                            _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info
                         )
                     )
 
@@ -5432,6 +5446,7 @@ def _bedrock_converse_messages_pt(
     deployment_model: str | None = None,
     base_model: str | None = None,
     drop_params: bool = False,
+    model_info: Mapping[str, object] | None = None,
 ) -> list[BedrockMessageBlock]:
     """
     Converts given messages from OpenAI format to Bedrock format
@@ -5500,6 +5515,7 @@ def _bedrock_converse_messages_pt(
                             block_type="content_block",
                             model=model,
                             deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if _cache_point_block is not None:
                             _parts.append(_cache_point_block)
@@ -5507,7 +5523,11 @@ def _bedrock_converse_messages_pt(
             elif message_content and isinstance(message_content, str):
                 _part = BedrockContentBlock(text=message_content)
                 _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
-                    message_block, block_type="content_block", model=model, deployment_model=deployment_model
+                    message_block,
+                    block_type="content_block",
+                    model=model,
+                    deployment_model=deployment_model,
+                    model_info=model_info,
                 )
                 user_content.append(_part)
                 if _cache_point_block is not None:
@@ -5560,6 +5580,7 @@ def _bedrock_converse_messages_pt(
                     block_type="content_block",
                     model=model,
                     deployment_model=deployment_model,
+                    model_info=model_info,
                 )
                 if cache_point_block is not None:
                     tool_content.append(cache_point_block)
@@ -5626,6 +5647,7 @@ def _bedrock_converse_messages_pt(
                             block_type="content_block",
                             model=model,
                             deployment_model=deployment_model,
+                            model_info=model_info,
                         )
                         if _cache_point_block is not None and not (
                             drop_params and _as_thinking_block(element) is not None
@@ -5638,14 +5660,20 @@ def _bedrock_converse_messages_pt(
                     assistant_content.append(BedrockContentBlock(text=_assistant_content))
                 # Add cache point block for assistant string content
                 _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
-                    assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model
+                    assistant_message_block,
+                    block_type="content_block",
+                    model=model,
+                    deployment_model=deployment_model,
+                    model_info=model_info,
                 )
                 if _cache_point_block is not None:
                     assistant_content.append(_cache_point_block)
             _tool_calls = assistant_message_block.get("tool_calls", [])
             if _tool_calls:
                 assistant_content.extend(
-                    _convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model)
+                    _convert_to_bedrock_tool_call_invoke(
+                        _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info
+                    )
                 )
 
             msg_i += 1
@@ -5701,7 +5729,10 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
 
 
 def add_cache_point_tool_block(
-    tool: dict, model: str | None = None, deployment_model: str | None = None
+    tool: dict,
+    model: str | None = None,
+    deployment_model: str | None = None,
+    model_info: Mapping[str, object] | None = None,
 ) -> BedrockToolBlock | None:
     from litellm.llms.bedrock.common_utils import (
         bedrock_cache_point_ttl_supported,
@@ -5709,7 +5740,7 @@ def add_cache_point_tool_block(
     )
 
     cache_control: Final = tool.get("cache_control", None)
-    if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model):
+    if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model, model_info):
         cache_point: Final = cache_control.get("type", "ephemeral")
         if cache_point == "ephemeral":
             cache_point_block: Final[CachePointBlock] = {"type": "default"}
@@ -5746,7 +5777,10 @@ def _is_bedrock_tool_block(tool: dict) -> bool:
 
 
 def _bedrock_tools_pt(
-    tools: list, model: str | None = None, deployment_model: str | None = None
+    tools: list,
+    model: str | None = None,
+    deployment_model: str | None = None,
+    model_info: Mapping[str, object] | None = None,
 ) -> list[BedrockToolBlock]:
     """
     OpenAI tools looks like:
@@ -5867,7 +5901,9 @@ def _bedrock_tools_pt(
         tool_block_list.append(tool_block)
 
         ## ADD CACHE POINT TOOL BLOCK ##
-        cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model)
+        cache_point_tool_block = add_cache_point_tool_block(
+            tool, model=model, deployment_model=deployment_model, model_info=model_info
+        )
         if cache_point_tool_block is not None:
             tool_block_list.append(cache_point_tool_block)
 
@@ -5915,8 +5951,13 @@ def function_call_prompt(
     llm_provider: str,
     drop_params: bool,
 ) -> list[dict[str, object]]:
+    prompt_functions: Final = tools_without_hosted_mcp(
+        functions, model=model, route=llm_provider, drop_params=drop_params
+    )
+    if not prompt_functions:
+        return messages
     header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
-    function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions)
+    function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions)
     if not any(_is_system_message(message) for message in messages):
         return [*messages, {"role": "system", "content": function_prompt}]
     return [

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -5,28 +5,33 @@ Helper functions to handle images passed in messages
 import asyncio
 import base64
 import time
-from collections.abc import Callable, Iterable, Mapping
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
-from itertools import chain
+from functools import partial
+from itertools import accumulate, chain, takewhile, tee
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeVar
+from urllib.parse import urlparse
 
 import httpx
-from httpx import Response
+from httpx import USE_CLIENT_DEFAULT, Response
 
 import litellm
 from litellm import verbose_logger
 from litellm.caching.caching import InMemoryCache
-from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
+from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT
 from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
 from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
 from litellm.types.llms.openai import AllMessageValues
 
 MAX_IMGS_IN_MEMORY: Final = 10
 MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20
+_Fetched = TypeVar("_Fetched")
 _FETCH_ATTEMPTS: Final = 3
 
 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
+pdf_probe_verdicts: Final = InMemoryCache()
 
 
 def _header_value(response: Response, name: str) -> str | None:
@@ -209,6 +214,173 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool:
     return media.part_type == "image_url"
 
 
+_PDF_MEDIA_TYPES: Final = frozenset({"application/pdf", "application/x-pdf"})
+_GENERIC_MEDIA_TYPES: Final = frozenset({"application/octet-stream", "binary/octet-stream"})
+_PDF_SIGNATURE: Final = b"%PDF-"
+_PDF_SIGNATURE_WINDOW: Final = 1024
+_PDF_PROBE_HEADERS: Final = MappingProxyType({"Range": f"bytes=0-{_PDF_SIGNATURE_WINDOW - 1}"})
+_PDF_PROBE_STATUSES: Final = frozenset({200, 206})
+_PDF_PROBE_FAILURES: Final = (httpx.HTTPError, httpx.InvalidURL, ValueError)
+_ASYNC_PDF_PROBE_FAILURES: Final = (*_PDF_PROBE_FAILURES, asyncio.TimeoutError)
+_PDF_PROBE_VERDICT_TTL: Final = 600
+_PDF_PROBE_FAILURE_TTL: Final = 60
+_DECODED_BODY_HEADERS: Final = frozenset({"content-encoding", "content-length", "transfer-encoding"})
+
+
+def _bare_media_type(media_type: str) -> str:
+    return media_type.split(";", 1)[0].strip().lower()
+
+
+def is_pdf_media_type(media_type: str) -> bool:
+    return _bare_media_type(media_type) in _PDF_MEDIA_TYPES
+
+
+def is_unformatted_pdf_link(url: str, image_format: str | None) -> bool:
+    lowered: Final = url.lower()
+    return not image_format and (urlparse(lowered).path.endswith(".pdf") or lowered.endswith(".pdf"))
+
+
+def is_unformatted_pdf_link_image(media: RemoteMedia) -> bool:
+    image_format: Final = media.fields.get("format")
+    return inline_remote_image_urls(media) and is_unformatted_pdf_link(
+        media.url, image_format if isinstance(image_format, str) else None
+    )
+
+
+def _serves_pdf(response: Response) -> bool:
+    if response.status_code not in _PDF_PROBE_STATUSES:
+        return False
+    served_type: Final = _bare_media_type(_header_value(response, "Content-Type") or "")
+    if served_type and served_type not in _GENERIC_MEDIA_TYPES:
+        return served_type in _PDF_MEDIA_TYPES
+    return _PDF_SIGNATURE in response.content[:_PDF_SIGNATURE_WINDOW]
+
+
+def _signature_window_chunks(chunks: Iterable[bytes], deadline: float) -> Iterator[bytes]:
+    reading, counting = tee(chunks)
+    read_so_far: Final = chain((0,), accumulate(map(len, counting)))
+    still_reading: Final = takewhile(
+        lambda total: total < _PDF_SIGNATURE_WINDOW and time.monotonic() < deadline, read_so_far
+    )
+    return (chunk for _, chunk in zip(still_reading, reading))
+
+
+def _with_body_prefix(response: Response, prefix: bytes) -> Response:
+    return Response(
+        response.status_code,
+        headers=[(name, value) for name, value in response.headers.multi_items() if name not in _DECODED_BODY_HEADERS],
+        content=prefix,
+        request=response.request,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _BodyPrefixReader:
+    client: httpx.Client
+    deadline: float
+
+    def get(
+        self,
+        url: str,
+        *,
+        headers: Mapping[str, str] | None = None,
+        follow_redirects: bool = False,
+        timeout: float | httpx.Timeout | None = None,
+    ) -> Response:
+        with self.client.stream(
+            "GET",
+            url,
+            headers=headers,
+            follow_redirects=follow_redirects,
+            timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
+        ) as response:
+            prefix: Final = b"".join(_signature_window_chunks(response.iter_bytes(), self.deadline))
+            return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW])
+
+
+@dataclass(frozen=True, slots=True)
+class _AsyncBodyPrefixReader:
+    client: httpx.AsyncClient
+
+    async def get(
+        self,
+        url: str,
+        *,
+        headers: Mapping[str, str] | None = None,
+        follow_redirects: bool = False,
+        timeout: float | httpx.Timeout | None = None,
+        max_response_bytes: int | None = None,
+    ) -> Response:
+        async with self.client.stream(
+            "GET",
+            url,
+            headers=headers,
+            follow_redirects=follow_redirects,
+            timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
+        ) as response:
+            return _with_body_prefix(response, await anext(response.aiter_bytes(max_response_bytes), b""))
+
+
+def _cached_pdf_verdict(url: str) -> bool | None:
+    cached: Final = pdf_probe_verdicts.get_cache(url)
+    return cached if isinstance(cached, bool) else None
+
+
+def _remember_pdf_verdict(url: str, verdict: bool, ttl: int) -> bool:
+    pdf_probe_verdicts.set_cache(url, verdict, ttl=ttl)
+    return verdict
+
+
+def _remember_probe_response(url: str, response: Response) -> bool:
+    ttl: Final = _PDF_PROBE_VERDICT_TTL if response.status_code in _PDF_PROBE_STATUSES else _PDF_PROBE_FAILURE_TTL
+    return _remember_pdf_verdict(url, _serves_pdf(response), ttl)
+
+
+def _failed_pdf_probe(url: str, error: Exception) -> bool:
+    verbose_logger.debug("PDF probe of %s failed, sending it as an image link: %s", url, error)
+    return _remember_pdf_verdict(url, False, _PDF_PROBE_FAILURE_TTL)
+
+
+def serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
+    if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
+        return False
+    cached: Final = _cached_pdf_verdict(url)
+    if cached is not None:
+        return cached
+    try:
+        response: Final = safe_get(
+            _BodyPrefixReader(litellm.module_level_client.client, deadline=time.monotonic() + timeout),
+            url,
+            headers=dict(_PDF_PROBE_HEADERS),
+            timeout=timeout,
+        )
+    except _PDF_PROBE_FAILURES as e:
+        return _failed_pdf_probe(url, e)
+    return _remember_probe_response(url, response)
+
+
+async def async_serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
+    if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
+        return False
+    cached: Final = _cached_pdf_verdict(url)
+    if cached is not None:
+        return cached
+    try:
+        response: Final = await asyncio.wait_for(
+            async_safe_get(
+                _AsyncBodyPrefixReader(litellm.module_level_aclient.client),
+                url,
+                headers=dict(_PDF_PROBE_HEADERS),
+                timeout=timeout,
+                max_response_bytes=_PDF_SIGNATURE_WINDOW,
+            ),
+            timeout,
+        )
+    except _ASYNC_PDF_PROBE_FAILURES as e:
+        return _failed_pdf_probe(url, e)
+    return _remember_probe_response(url, response)
+
+
 def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None:
     if fields.get("type") != "image_url":
         return None
@@ -292,15 +464,40 @@ def _inline_part(part: object, data_urls: Mapping[str, str], should_inline: Call
     return _inline(remote, data_url) if data_url is not None else part
 
 
-def _inline_message(
-    message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
-) -> AllMessageValues:
+def _with_parts(message: AllMessageValues, rewrite: Callable[[object], object]) -> AllMessageValues:
     parts: Final = _content_parts(message)
     if not parts:
         return message
-    inlined_parts: Final = [_inline_part(part, data_urls, should_inline) for part in parts]
-    inlined_message: Final = {**message, "content": inlined_parts}
-    return inlined_message  # pyright: ignore[reportReturnType]  # the same message with its remote parts inlined
+    rewritten: Final = {**message, "content": [rewrite(part) for part in parts]}
+    return rewritten  # pyright: ignore[reportReturnType]  # the same message with some of its parts rewritten
+
+
+def _inline_message(
+    message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
+) -> AllMessageValues:
+    return _with_parts(message, partial(_inline_part, data_urls=data_urls, should_inline=should_inline))
+
+
+def _pdf_formatted_part(part: object, pdf_links: frozenset[str], should_probe: Callable[[RemoteMedia], bool]) -> object:
+    remote: Final = _parse_remote_part(part)
+    match remote:
+        case _RemoteImage(fields, image_url, url) if url in pdf_links and should_probe(_remote_media(remote)):
+            image_url_fields: Final = image_url if image_url is not None else _NO_FIELDS
+            return {**fields, "image_url": {**image_url_fields, "url": url, "format": "application/pdf"}}
+        case _:
+            return part
+
+
+def _with_pdf_formats(
+    messages: Iterable[AllMessageValues],
+    probed: Iterable[tuple[str, bool]],
+    should_probe: Callable[[RemoteMedia], bool],
+) -> tuple[AllMessageValues, ...]:
+    pdf_links: Final = frozenset(url for url, is_pdf in probed if is_pdf)
+    if not pdf_links:
+        return tuple(messages)
+    rewrite: Final = partial(_pdf_formatted_part, pdf_links=pdf_links, should_probe=should_probe)
+    return tuple(_with_parts(message, rewrite) for message in messages)
 
 
 async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str:
@@ -308,11 +505,12 @@ async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float
         return await async_convert_url_to_base64(url, timeout=timeout)
 
 
-async def async_convert_urls_to_base64(
-    remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
-) -> tuple[str, ...]:
-    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
-    fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
+async def _serves_pdf_within(url: str, in_flight: asyncio.Semaphore) -> bool:
+    async with in_flight:
+        return await async_serves_pdf(url)
+
+
+async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -> tuple[_Fetched, ...]:
     try:
         return tuple(await asyncio.gather(*fetches))
     except BaseException:
@@ -322,6 +520,15 @@ async def async_convert_urls_to_base64(
         raise
 
 
+async def async_convert_urls_to_base64(
+    remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
+) -> tuple[str, ...]:
+    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
+    return await _gather_or_cancel_all(
+        tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
+    )
+
+
 def _remote_urls_to_inline(
     messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
 ) -> tuple[str, ...]:
@@ -351,3 +558,27 @@ async def async_inline_remote_media(
     data_urls: Final = await async_convert_urls_to_base64(remote_urls)
     inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))
     return [_inline_message(message, inlined, should_inline) for message in messages]
+
+
+def format_pdf_links(
+    messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
+) -> tuple[AllMessageValues, ...]:
+    links: Final = _remote_urls_to_inline(messages, should_probe)
+    if not links:
+        return tuple(messages)
+    with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, len(links))) as probes:
+        verdicts: Final = tuple(probes.map(serves_pdf, links))
+    return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)
+
+
+async def async_format_pdf_links(
+    messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
+) -> tuple[AllMessageValues, ...]:
+    links: Final = _remote_urls_to_inline(messages, should_probe)
+    if not links:
+        return tuple(messages)
+    in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
+    verdicts: Final = await _gather_or_cancel_all(
+        tuple(asyncio.create_task(_serves_pdf_within(url, in_flight)) for url in links)
+    )
+    return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -85,6 +85,11 @@ def message_field(message: object, key: str) -> object:
     return mapping.get(key) if mapping is not None else getattr(message, key, None)
 
 
+def string_field(message: object, key: str) -> str | None:
+    value: Final = message_field(message, key)
+    return value if isinstance(value, str) else None
+
+
 def is_system_message(message: object) -> bool:
     return message_field(message, "role") == "system"