Merge remote-tracking branch 'origin/main' into litellm_lakera_v2_bedrock_passthrough

.circleci/AGENTS.md

@@ -0,0 +1,7 @@
+# .circleci
+
+The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
+
+Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
+
+A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base

.circleci/config.yml

@@ -21,19 +21,30 @@ orbs:
   win: circleci/windows@5.0 # Add Windows orb
 
 commands:
-  checkout_migration_source:
+  checkout_over_https:
+    parameters:
+      revision:
+        type: string
+        default: ""
+      shell:
+        type: string
+        default: /bin/bash -eo pipefail
     steps:
       - run:
-          name: Select the requested migration test revision
+          name: Check out the public repository over HTTPS
+          shell: << parameters.shell >>
           environment:
-            MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
+            REQUESTED_REVISION: << parameters.revision >>
           command: |
-            revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
+            revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
             [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
             git init
             git remote add origin https://github.com/BerriAI/litellm.git
-            git fetch --depth 1 origin "$revision"
-            git checkout --detach FETCH_HEAD
+            for attempt in 1 2 3; do
+              git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
+              [[ $attempt -lt 3 ]] || exit 1
+              sleep 5
+            done
   skip_if_unrelated_changes:
     parameters:
       category:
@@ -347,7 +358,7 @@ commands:
       - when:
           condition: << parameters.checkout >>
           steps:
-            - checkout
+            - checkout_over_https
       - setup_google_dns
       - install_uv
       - install_rust
@@ -377,7 +388,8 @@ jobs:
       CARGO_HTTP_MULTIPLEXING: "false"
       CARGO_NET_RETRY: "5"
     steps:
-      - checkout
+      - checkout_over_https:
+          shell: bash.exe
       - run:
           name: Skip job when no windows-release-relevant files changed
           shell: bash.exe
@@ -451,7 +463,7 @@ jobs:
           password: ${DOCKERHUB_PASSWORD}
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - setup_google_dns
       - install_uv
       - install_rust
@@ -482,7 +494,7 @@ jobs:
     working_directory: ~/project
 
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -559,7 +571,7 @@ jobs:
     resource_class: medium
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -659,7 +671,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -740,7 +752,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
@@ -794,7 +806,7 @@ jobs:
       # the rest of the suite: nothing else performs a logout.
       PROXY_LOGOUT_URL: "https://www.example.com"
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -940,7 +952,7 @@ jobs:
       # and by migration.serverRootPath.config.ts, which refuses to run without it.
       SERVER_ROOT_PATH: "/litellm"
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -1066,11 +1078,12 @@ jobs:
       - when:
           condition: << parameters.migration_qualification >>
           steps:
-            - checkout_migration_source
+            - checkout_over_https:
+                revision: << pipeline.parameters.migration_source_sha >>
       - unless:
           condition: << parameters.migration_qualification >>
           steps:
-            - checkout
+            - checkout_over_https
             - skip_if_unrelated_changes
 
       - run:
@@ -1122,7 +1135,8 @@ jobs:
       MIGRATION_TEST_OUTPUT: /tmp/migration-results
       PYTHONPATH: tests/e2e
     steps:
-      - checkout_migration_source
+      - checkout_over_https:
+          revision: << pipeline.parameters.migration_source_sha >>
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -1197,7 +1211,7 @@ jobs:
     resource_class: medium
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -1371,7 +1385,7 @@ jobs:
     environment:
       UV_PYTHON_DOWNLOADS: never
     steps:
-      - checkout
+      - checkout_over_https
       - install_uv
       - run:
           name: Install the locked shard verifier and coverage dependencies
@@ -1419,7 +1433,7 @@ jobs:
             ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
           command: |
             [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
-            git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
+            git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
             git rm -r -f --quiet litellm enterprise litellm-proxy-extras
             git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
             git reset --quiet
@@ -1494,7 +1508,7 @@ jobs:
     environment:
       DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - setup_litellm_test_deps:
           checkout: false

.circleci/scripts/run_integration.sh

@@ -128,6 +128,8 @@ fi
 sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
 awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
 
+export UV_OFFLINE=true
+
 if [ "$mode" = parity ]; then
   export INTEGRATION_WORKERS=1
 else

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -4,9 +4,21 @@
      Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
      "## Affected release" with nothing under it must not appear in the final description -->
 
-<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
+## Handwritten 1-3 sentence description
+
+<!-- The author writes a plain summary in their own words before requesting review.
+     Wrap newly AI-written summary text in the make-me-human-readable tags below, including backticks.
+     The author reviews or rewrites the draft and removes the tags.
+     When updating an existing PR, preserve the author's wording and removed tags.
+     Changes elsewhere in the body do not make an unchanged summary an AI draft again.
+     Add tags again only around newly AI-written replacement summary text, never merely to match this template.
+     This preserves author review across later updates (2026-11-02, #49779).
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
-     No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
+     No bullets, tables, or code names here. Everything else goes inside the Details block below -->
+
+`<make-me-human-readable>`
+<blah>
+`</make-me-human-readable>`
 
 <details>
 <summary>Details</summary>

.github/workflows/compat-matrix-image.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - tests/e2e/claude_code/cron_vm/**
-      - tests/e2e/claude_code/pr_gate_version_resolver.py
+      - tests/e2e/cli_harness/npm_version_resolver.py
       - .github/workflows/compat-matrix-image.yml
   workflow_dispatch:
 
@@ -36,7 +36,7 @@ jobs:
             whoami
             gh --version
             uv --version
-            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
+            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
             /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
             /tmp/claude-cli/claude --version
           '

.github/workflows/create-rc-branch.yml

@@ -2,7 +2,7 @@ name: Create RC Branch
 
 on:
   schedule:
-    - cron: "0 3 * * 5"
+    - cron: "0 3 * * 1"
       timezone: "America/Los_Angeles"
   workflow_dispatch:
 

.github/workflows/deploy-dev-on-main.yml

@@ -1,30 +0,0 @@
-name: Deploy main to gateway-dev
-
-on:
-  push:
-    branches: [main]
-
-permissions: {}
-
-concurrency:
-  group: deploy-dev-dispatch
-  cancel-in-progress: true
-
-jobs:
-  dispatch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
-        id: token
-        with:
-          app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
-          private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
-          owner: BerriAI
-          repositories: litellm-ops
-          permission-actions: write
-      - name: Dispatch the litellm-ops dev deploy
-        env:
-          GH_TOKEN: ${{ steps.token.outputs.token }}
-        run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

.github/workflows/test-linting.yml

@@ -208,7 +208,7 @@ jobs:
             exit 0
           fi
           retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
+          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
           tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
           PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
           uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \

.github/workflows/test-unit.yml

@@ -441,6 +441,7 @@ jobs:
               tests/unit/ocr
               tests/unit/passthrough
               tests/unit/rag
+              tests/unit/cookbook
               tests/unit/realtime_api
               tests/unit/rerank_api
               tests/unit/search

.github/workflows/weekly_load_anomaly.yml

@@ -1,85 +0,0 @@
-name: "Weekly Load Anomaly Check"
-
-on:
-  schedule:
-    - cron: "0 12 * * 6"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  weekly-load-anomaly:
-    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    services:
-      postgres:
-        image: postgres:16.6
-        env:
-          POSTGRES_USER: llmproxy
-          POSTGRES_PASSWORD: dbpassword9090
-          POSTGRES_DB: litellm
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U llmproxy"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
-    env:
-      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
-      LITELLM_MASTER_KEY: sk-weekly-anomaly-check
-      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache the Rust build
-        uses: ./.github/actions/cache-cargo-build
-
-      - name: Install dependencies
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
-
-      - name: Cache Prisma binaries
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Start the proxy
-        run: |
-          nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
-          for _ in $(seq 1 90); do
-            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "proxy never became live"
-          tail -n 100 proxy.log
-          exit 1
-
-      - name: Run the weekly session anomaly test
-        env:
-          E2E_WEEKLY_ANOMALY: "1"
-        run: |
-          uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
-
-      - name: Show proxy log on failure
-        if: failure()
-        run: tail -n 300 proxy.log

AGENTS.md

@@ -41,6 +41,8 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
+Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`, and swap its `--config` for a copy of `litellm/proxy/dev_config.yaml` that points both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. Check that two worker PIDs are serving, and use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload`. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker check for it. Say in the proof which part you dropped and why
+
 Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
 
 For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce

README.md

@@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) |  |  |  | ✅ |  |  |  |  |  |  |
 | [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) |  |  |  |  |  |  |  |  |  |  |
 | [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -696,7 +697,7 @@ For companies that need better security, user management and professional suppor
 [Talk to founders](https://enterprise.litellm.ai/demo)
 
 This covers:
-- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
+- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
 - ✅ **Professional Support - Dedicated discord + slack**

cookbook/gpt_realtime_translate.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+import sys
+import wave
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlencode, urlsplit, urlunsplit
+
+import websockets
+from websockets.asyncio.client import ClientConnection
+
+SAMPLE_RATE = 24_000
+CHANNELS = 1
+SAMPLE_WIDTH = 2
+CHUNK_DURATION_SECONDS = 0.1
+CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
+RESPONSE_TIMEOUT_SECONDS = 30.0
+AUDIO_EVENT_TYPES = frozenset(
+    {
+        "session.output_audio.delta",
+        "response.audio.delta",
+        "response.output_audio.delta",
+    }
+)
+TRANSCRIPT_EVENT_TYPES = frozenset(
+    {
+        "session.output_transcript.delta",
+        "response.text.delta",
+        "response.output_audio_transcript.delta",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    input_wav: Path
+    output_wav: Path
+    base_url: str
+    model: str
+    target_language: str
+    trailing_silence_seconds: float
+    api_key: str
+
+
+def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
+    sys.stdout.write(f"{message}{end}")
+    if flush:
+        sys.stdout.flush()
+
+
+def write_stderr(message: str) -> None:
+    sys.stderr.write(f"{message}\n")
+
+
+def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
+    parser = argparse.ArgumentParser(
+        description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
+    )
+    parser.add_argument("input_wav", type=Path)
+    parser.add_argument("--output", type=Path, default=Path("translated.wav"))
+    parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
+    parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
+    parser.add_argument("--target-language", default="fr")
+    parser.add_argument("--trailing-silence", type=float, default=1.5)
+    parsed = parser.parse_args(argv)
+    api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
+    if parsed.trailing_silence < 0:
+        return "--trailing-silence must be zero or greater"
+    return Settings(
+        input_wav=parsed.input_wav,
+        output_wav=parsed.output,
+        base_url=parsed.base_url,
+        model=parsed.model,
+        target_language=parsed.target_language,
+        trailing_silence_seconds=parsed.trailing_silence,
+        api_key=api_key,
+    )
+
+
+def translation_url(base_url: str, model: str) -> str | None:
+    parsed = urlsplit(base_url.rstrip("/"))
+    scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
+    if not scheme or not parsed.netloc:
+        return None
+    base_path = parsed.path.rstrip("/")
+    realtime_path = (
+        f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
+    )
+    return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
+
+
+def read_pcm16_wav(path: Path) -> bytes | str:
+    try:
+        with wave.open(str(path), "rb") as source:
+            actual_format = (
+                source.getnchannels(),
+                source.getsampwidth(),
+                source.getframerate(),
+                source.getcomptype(),
+            )
+            expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
+            if actual_format != expected_format:
+                return (
+                    f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
+                    f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
+                    f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
+                )
+            return source.readframes(source.getnframes())
+    except (OSError, EOFError, wave.Error) as exc:
+        return f"Unable to read {path}: {exc}"
+
+
+def audio_chunks(audio: bytes) -> Iterator[bytes]:
+    return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
+
+
+def audio_message(audio: bytes) -> str:
+    return json.dumps(
+        {
+            "type": "session.input_audio_buffer.append",
+            "audio": base64.b64encode(audio).decode("ascii"),
+        }
+    )
+
+
+async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
+    await connection.send(
+        json.dumps(
+            {
+                "type": "session.update",
+                "session": {"audio": {"output": {"language": target_language}}},
+            }
+        )
+    )
+    while True:
+        raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
+        event = json.loads(raw_event)
+        event_type = event.get("type")
+        if event_type == "session.created":
+            write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
+        if event_type == "session.updated":
+            return None
+        if event_type == "error":
+            return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
+
+
+async def send_audio(
+    connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
+) -> None:
+    silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
+    try:
+        for chunk in audio_chunks(pcm + silence):
+            await connection.send(audio_message(chunk))
+            await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
+        await connection.send(json.dumps({"type": "session.close"}))
+    finally:
+        finished.set()
+
+
+async def receive_translation(
+    connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
+) -> str | None:
+    audio_received = asyncio.Event()
+    try:
+        with wave.open(str(output_path), "wb") as output:
+            output.setnchannels(CHANNELS)
+            output.setsampwidth(SAMPLE_WIDTH)
+            output.setframerate(SAMPLE_RATE)
+            write_stdout("Translation: ", end="", flush=True)
+            while True:
+                try:
+                    raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
+                except TimeoutError:
+                    if not sender_finished.is_set():
+                        continue
+                    return (
+                        f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
+                    )
+                event = json.loads(raw_event)
+                event_type = event.get("type")
+                if event_type in AUDIO_EVENT_TYPES:
+                    output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
+                    audio_received.set()
+                elif event_type in TRANSCRIPT_EVENT_TYPES:
+                    write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
+                elif event_type == "error":
+                    return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
+                elif event_type == "session.closed":
+                    write_stdout()
+                    return None if audio_received.is_set() else "The translation stream ended without translated audio"
+    except (OSError, wave.Error) as exc:
+        return f"Unable to write {output_path}: {exc}"
+
+
+async def translate(settings: Settings, pcm: bytes) -> str | None:
+    url = translation_url(settings.base_url, settings.model)
+    if not url:
+        return f"Invalid --base-url: {settings.base_url}"
+    sender_finished = asyncio.Event()
+    try:
+        async with websockets.connect(
+            url,
+            additional_headers={"Authorization": f"Bearer {settings.api_key}"},
+            proxy=None,
+            open_timeout=20,
+            close_timeout=5,
+        ) as connection:
+            configuration_error = await configure_session(connection, settings.target_language)
+            if configuration_error:
+                return configuration_error
+            async with asyncio.TaskGroup() as tasks:
+                receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
+                tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
+            return receiver.result()
+    except Exception as exc:
+        return f"Translation failed: {type(exc).__name__}: {exc}"
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    settings = parse_args(argv)
+    if isinstance(settings, str):
+        write_stderr(settings)
+        return 2
+    pcm = read_pcm16_wav(settings.input_wav)
+    if isinstance(pcm, str):
+        write_stderr(pcm)
+        return 2
+    error = asyncio.run(translate(settings, pcm))
+    if error:
+        write_stderr(error)
+        return 1
+    write_stdout(f"Translated audio: {settings.output_wav}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -200,6 +200,7 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
     unpriced: Final = _without_read_price(billed.response)
     cost: Final = logging_obj.response_cost_calculator(result=unpriced)
     end_time: Final = datetime.now()
+    logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
         async_coroutine=logging_obj.async_success_handler(
             result=unpriced, start_time=logging_obj.start_time, end_time=end_time

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -3,9 +3,11 @@
 
 import asyncio
 import base64
+import copy
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from itertools import chain
 from types import MappingProxyType
@@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
 
 import litellm
 from litellm import Router, verbose_logger
-from litellm._internal_context import with_service_target
+from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.constants import (
@@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
     request_tags_from_metadata,
 )
 from litellm.repositories.managed_file_repository import ManagedFileRepository
+from litellm.responses.streaming_iterator import spawn_logging_task
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccessIssue]
     AllMessageValues,
@@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
+_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
+_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderUpload:
+    model_id: str
+    provider_file_id: str
+    upload_call_id: str | None = None
+
+
+def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
+    file_hidden_params: Final = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
+        dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
+    )
+    model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
+        file_hidden_params.get("model_file_id_mapping") or {}
+    )
+    upload_call_id: Final = file_hidden_params.get("litellm_call_id")
+    return tuple(
+        _ProviderUpload(
+            model_id=model_id,
+            provider_file_id=provider_file_id,
+            upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
+        )
+        for model_id, provider_file_id in model_file_id_mapping.items()
+    )
+
+
+def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
+    credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
+    return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
+
+
+def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
+    upload: Final = uploads[index]
+    other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
+    return all(other.model_id != upload.model_id for other in other_uploads)
+
+
+def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
+    trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
+    provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
+    return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
+
+
+def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
+    decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
+    carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
+    if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
+        return None
+    return carried
+
+
+async def _delete_provider_file(
+    llm_router: Router,
+    model_id: str,
+    provider_file_id: str,
+    request_data: Mapping[str, object],
+) -> None:
+    credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
+    delete_data: Final = {
+        **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
+        **(
+            {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
+            if credentials is not None
+            else {}
+        ),
+    }
+    await llm_router.afile_delete(
+        model=model_id,
+        file_id=provider_file_id,
+        **cast(_RouterFileCallKwargs, delete_data),
+    )
+
+
+def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
+    return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
+
+
+async def _delete_redundant_upload(
+    llm_router: Router,
+    upload: _ProviderUpload,
+    delete_request: Mapping[str, object],
+    parent_call_id: str | None,
+) -> None:
+    try:
+        with internal_sub_call(parent_call_id):
+            await _delete_provider_file(
+                llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
+            )
+    except Exception as error:
+        verbose_logger.warning(
+            f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
+            f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
+            f"{type(error).__name__} {sanitize_for_log(error)}"
+        )
+
+
+async def _delete_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    delete_request: Mapping[str, object],
+) -> None:
+    await asyncio.gather(
+        *(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
+    )
+
+
+def _start_deleting_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    create_file_request: CreateFileRequest,
+) -> None:
+    if not uploads:
+        return
+    delete_request: Final = MappingProxyType(
+        {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
+    )
+    with post_response_phase():
+        spawn_logging_task(
+            asyncio.get_running_loop(),
+            _delete_redundant_uploads(llm_router, uploads, delete_request),
+            task_name="managed file repeat upload cleanup",
+        )
+
+
 def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
     return prisma_client.db
 
@@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             target_model_names_list=target_model_names_list,
         )
 
-        ## STORE MODEL MAPPINGS IN DB
-        model_mappings: Dict[str, str] = {}
-
-        for file_object in responses:
-            file_hidden_params = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
-                dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
-            )
-            model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
-            if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
-                model_mappings.update(model_file_id_mapping)
+        uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
+        per_target_model_ids: Final = frozenset(
+            model_id
+            for model_id in {upload.model_id for upload in uploads}
+            if _keeps_one_upload_per_target(llm_router, model_id)
+        )
+        kept_uploads: Final = tuple(
+            upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
+        )
+        model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
 
         await self.store_unified_file_id(
             file_id=response.id,
@@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             model_mappings=model_mappings,
             user_api_key_dict=user_api_key_dict,
         )
+        _start_deleting_redundant_uploads(
+            llm_router,
+            tuple(
+                upload
+                for upload in uploads
+                if upload not in kept_uploads and upload.model_id not in per_target_model_ids
+            ),
+            create_file_request,
+        )
 
         # Emit Prometheus metrics for managed file creation
         prom_logger = self._get_prometheus_logger()
@@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
         if not specific_model_file_id_mapping:
             return
-        filtered_data: Final = {
-            k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
-        }
         for model_id, model_file_id in specific_model_file_id_mapping.items():
-            credentials = llm_router.registry().credentials(model_id, form="with_provider")
-            delete_data = {
-                **filtered_data,
-                **(
-                    {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
-                    if credentials is not None
-                    else {}
-                ),
-            }
-            await llm_router.afile_delete(
-                model=model_id,
-                file_id=model_file_id,
-                **cast(_RouterFileCallKwargs, delete_data),
+            await _delete_provider_file(llm_router, model_id, model_file_id, data)
+        unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
+        delete_call_id: Final = data.get("litellm_call_id")
+        if unrecorded_upload is not None:
+            await _delete_redundant_upload(
+                llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
             )
 
     async def afile_content(

litellm-rust/crates/callbacks-legacy-python/python_contract.json

@@ -33,6 +33,17 @@
     "api_key",
     "additional_args"
   ],
+  "record_response": [
+    "logger",
+    "status",
+    "headers",
+    "body"
+  ],
+  "record_api_window": [
+    "logger",
+    "start",
+    "end"
+  ],
   "defers_async_logging": [
     "logger"
   ],
@@ -91,6 +102,13 @@
     "logger",
     "head"
   ],
+  "stamp_stream_head": [
+    "head",
+    "logger",
+    "kwargs",
+    "start_time",
+    "end_time"
+  ],
   "stream_success": [
     "logger",
     "url_route",

litellm-rust/crates/callbacks-legacy-python/src/adapter.rs

@@ -57,6 +57,8 @@ pub struct LegacyLogging {
     asynchronous: bool,
     internal: bool,
     cache_key: Option<String>,
+    /// When the provider request went out, until its window is recorded (Messages only).
+    api_started: Option<f64>,
 }
 
 fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
@@ -87,6 +89,60 @@ impl LegacyLogging {
             asynchronous,
             internal: false,
             cache_key: None,
+            api_started: None,
+        }
+    }
+
+    /// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as
+    /// its own argument, so the logger never sees the caller's raw metadata, only the validated
+    /// request metadata merged in through `litellm_params`. Python updates the logger once, before
+    /// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the
+    /// logged metadata is not rebuilt from what the call wrote into it since.
+    fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult<Bound<'py, PyDict>> {
+        let kwargs = self.call.kwargs().bind(py).copy()?;
+        if self.operation != LoggingOperation::Messages {
+            return Ok(kwargs);
+        }
+        let consumed: &[&str] = if refresh {
+            &["metadata", "litellm_metadata"]
+        } else {
+            &["metadata"]
+        };
+        for name in consumed {
+            if kwargs.contains(name)? {
+                kwargs.del_item(name)?;
+            }
+        }
+        Ok(kwargs)
+    }
+
+    /// Python's Messages handler hands the logger its request params, validated `metadata`
+    /// included, as `litellm_params`, beside the `aanthropic_messages` call flag.
+    fn logged_request_params(
+        &self,
+        context: &RequestContext,
+    ) -> Option<serde_json::Map<String, Value>> {
+        if self.operation != LoggingOperation::Messages {
+            return None;
+        }
+        let mut params = context
+            .optional_params
+            .as_object()
+            .cloned()
+            .unwrap_or_default();
+        params.insert(
+            "aanthropic_messages".to_owned(),
+            Value::Bool(self.asynchronous),
+        );
+        Some(params)
+    }
+
+    /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync
+    /// client is untimed.
+    fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> {
+        match self.api_started.take() {
+            Some(start) => self.logger()?.record_api_window(py, start, end),
+            None => Ok(()),
         }
     }
 
@@ -111,6 +167,16 @@ impl LegacyLogging {
         }
     }
 
+    /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key.
+    fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> {
+        match self.operation {
+            LoggingOperation::Messages => Some(""),
+            LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => {
+                context.api_key.as_ref().map(|api_key| api_key.expose())
+            }
+        }
+    }
+
     fn stream_billing(&self) -> Option<PassThroughStream> {
         match self.operation {
             LoggingOperation::Messages => Some(PassThroughStream {
@@ -377,7 +443,13 @@ impl LegacyLogging {
         context: &RequestContext,
     ) -> PyResult<HookStep<Self, Box<WireRequest>>> {
         let logger = self.logger()?;
-        logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?;
+        logger.update_from_kwargs(
+            py,
+            &self.logging_kwargs(py, false)?,
+            &wire,
+            context,
+            self.logged_request_params(context).as_ref(),
+        )?;
         let body = to_py(py, &wire.body)?
             .into_bound(py)
             .cast_into::<PyDict>()?;
@@ -400,11 +472,14 @@ impl LegacyLogging {
         self.logger()?.pre_call(
             py,
             self.input_description(),
-            context.api_key.as_ref().map(|api_key| api_key.expose()),
+            self.logged_api_key(context),
             &body,
             &headers,
             &wire.url,
         )?;
+        if self.operation == LoggingOperation::Messages && self.asynchronous {
+            self.api_started = Some(epoch_seconds());
+        }
         let headers = headers
             .iter()
             .map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
@@ -455,7 +530,7 @@ impl LegacyLogging {
             py,
             (
                 &logger,
-                self.call.kwargs(),
+                self.logging_kwargs(py, true)?,
                 &facts.provider.model,
                 logger.getattr("optional_params")?,
                 params,
@@ -483,6 +558,16 @@ impl LegacyLogging {
         py: Python<'_>,
         raw: &RawResponse,
     ) -> PyResult<HookStep<Self, ()>> {
+        if self.operation == LoggingOperation::Messages {
+            // Python's Messages handler never calls `post_call` at receipt: it stores the raw
+            // response, and logging replays it through `AnthropicConfig.transform_response`.
+            self.close_api_window(py, epoch_seconds())?;
+            if let Some(head) = &raw.head {
+                self.logger()?
+                    .record_response(py, head.status, &head.headers, &raw.body)?;
+            }
+            return Ok(HookStep::Ready(()));
+        }
         let api_key = self
             .request
             .as_ref()
@@ -522,6 +607,9 @@ impl LegacyLogging {
     ) -> PyResult<HookStep<Self, ()>> {
         self.end = Some(datetime(py, timing.end_time)?);
         self.error = Some(error.clone_ref(py).into_value(py));
+        if self.logger.is_some() {
+            self.close_api_window(py, timing.end_time)?;
+        }
         if self.stream.is_some() {
             return self.stream_failure(py);
         }
@@ -543,6 +631,20 @@ impl LegacyLogging {
             head.bind(py).set_item("cache_key", key)?;
             head.bind(py).set_item("cache_hit", true)?;
         }
+        self.close_api_window(py, epoch_seconds())?;
+        if self.operation == LoggingOperation::Messages {
+            // Python's `@client` stamps the call metadata onto the stream it hands back.
+            Streaming::StampHead.call(
+                py,
+                (
+                    head.bind(py),
+                    self.logger()?.object(py),
+                    self.call.kwargs(),
+                    &self.start,
+                    datetime(py, epoch_seconds())?,
+                ),
+            )?;
+        }
         Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
         self.stream = Some(DeliveredStream {
             chunks: PyList::empty(py).unbind(),
@@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update
         let step = logging
             .before_provider_request(py, Box::new(wire), context)
             .unwrap();
-        let raw = RawResponse {
-            body: "raw response".into(),
-        };
+        let raw = RawResponse::new("raw response".into());
         assert!(matches!(
             logging
                 .on_event(
@@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response
                 py,
                 &locals,
                 c"
-assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
+assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls
 assert logger.calls[0][1] is head
-chunks = logger.calls[1][1]
+assert logger.calls[1][1] is head
+chunks = logger.calls[2][1]
 assert len(chunks) == 2
 assert chunks[0] is first
 assert chunks[1] is last

litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs

@@ -5,6 +5,7 @@
 use litellm_host::interceptors::{RequestContext, WireRequest};
 use litellm_host_python::to_py;
 use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
+use serde_json::Value;
 
 use crate::logger::PythonLogger;
 use crate::python::{Logging, Wrapper};
@@ -15,9 +16,10 @@ pub trait LegacyCallbacks {
     fn update_from_kwargs(
         &self,
         py: Python<'_>,
-        kwargs: &Py<PyDict>,
+        kwargs: &Bound<'_, PyDict>,
         wire: &WireRequest,
         context: &RequestContext,
+        request_params: Option<&serde_json::Map<String, Value>>,
     ) -> PyResult<()>;
 
     /// `Logging.pre_call`.
@@ -31,6 +33,20 @@ pub trait LegacyCallbacks {
         url: &str,
     ) -> PyResult<()>;
 
+    /// Records the provider's raw HTTP response on the logger, as Python's Messages handler
+    /// does with `httpx_response` and `response_headers`.
+    fn record_response(
+        &self,
+        py: Python<'_>,
+        status: u16,
+        headers: &[(String, String)],
+        body: &str,
+    ) -> PyResult<()>;
+
+    /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post`
+    /// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`.
+    fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>;
+
     /// `Logging.post_call`.
     fn post_call(
         &self,
@@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger {
     fn update_from_kwargs(
         &self,
         py: Python<'_>,
-        kwargs: &Py<PyDict>,
+        kwargs: &Bound<'_, PyDict>,
         wire: &WireRequest,
         context: &RequestContext,
+        request_params: Option<&serde_json::Map<String, Value>>,
     ) -> PyResult<()> {
         let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect();
-        let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?;
+        let redacted_kwargs = redact(py, kwargs, &secret_fields)?;
         let optional_params = redact(
             py,
             &to_py(py, &context.optional_params)?
@@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger {
             &secret_fields,
         )?;
         let params = PyDict::new(py);
-        params.set_item(
-            "litellm_call_id",
-            kwargs.bind(py).get_item("litellm_call_id")?,
-        )?;
+        params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
         params.set_item("api_base", &wire.url)?;
         for name in ["logger_fn", "litellm_request_debug"] {
-            if let Some(value) = kwargs.bind(py).get_item(name)? {
+            if let Some(value) = kwargs.get_item(name)? {
                 params.set_item(name, value)?;
             }
         }
         for name in custom_pricing_fields(py)? {
-            if let Some(value) = kwargs.bind(py).get_item(&name)?
+            if let Some(value) = kwargs.get_item(&name)?
                 && !value.is_none()
             {
                 params.set_item(name, value)?;
             }
         }
+        for (name, value) in request_params.into_iter().flatten() {
+            params.set_item(name, to_py(py, value)?)?;
+        }
         Logging::Update.call(
             py,
             (
@@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger {
         Ok(())
     }
 
+    fn record_response(
+        &self,
+        py: Python<'_>,
+        status: u16,
+        headers: &[(String, String)],
+        body: &str,
+    ) -> PyResult<()> {
+        Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?;
+        Ok(())
+    }
+
+    fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> {
+        Logging::RecordApiWindow.call(py, (self.object(py), start, end))?;
+        Ok(())
+    }
+
     fn post_call(
         &self,
         py: Python<'_>,

litellm-rust/crates/callbacks-legacy-python/src/python.rs

@@ -44,6 +44,10 @@ pub(crate) enum Logging {
     PreCall,
     #[strum(serialize = "post_call")]
     PostCall,
+    #[strum(serialize = "record_response")]
+    RecordResponse,
+    #[strum(serialize = "record_api_window")]
+    RecordApiWindow,
     #[strum(serialize = "defers_async_logging")]
     DefersAsync,
     #[strum(serialize = "defer_success")]
@@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks {
 pub(crate) enum Streaming {
     #[strum(serialize = "stream_opened")]
     Opened,
+    #[strum(serialize = "stamp_stream_head")]
+    StampHead,
     #[strum(serialize = "stream_success")]
     Success,
     #[strum(serialize = "stream_failure")]

litellm-rust/crates/callbacks-legacy-python/src/test_support.rs

@@ -65,6 +65,12 @@ FAKES = {
     'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call(
         original_response, api_key, additional_args
     ),
+    'record_response': lambda logger, status, headers, body: logger.model_call_details.update(
+        response_headers=dict(headers)
+    ),
+    'record_api_window': lambda logger, start, end: logger.model_call_details.update(
+        llm_api_duration_ms=(end - start) * 1000
+    ),
     'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)),
     'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending),
     'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls(
@@ -85,6 +91,7 @@ FAKES = {
     ),
     'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
     'stream_opened': lambda logger, head: logger.record('stream_opened', head),
+    'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head),
     'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
         'stream_success', list(chunks)
     ),

litellm-rust/crates/core-utils/src/core_helpers.rs

@@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
     ("max_tokens", "length"),
     ("refusal", "content_filter"),
     ("compaction", "length"),
+    ("pause_turn", "length"),
     ("model_context_window_exceeded", "length"),
     ("guardrail_intervened", "content_filter"),
     ("content_filtered", "content_filter"),
@@ -75,6 +76,7 @@ mod tests {
     #[case::end_turn("end_turn", "stop")]
     #[case::stop_sequence("stop_sequence", "stop")]
     #[case::max_tokens("max_tokens", "length")]
+    #[case::pause_turn("pause_turn", "length")]
     #[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
     #[case::refusal("refusal", "content_filter")]
     #[case::guardrail_intervened("guardrail_intervened", "content_filter")]

litellm-rust/crates/cost/AGENTS.md

@@ -0,0 +1 @@
+The price vocabulary (meter, band, channel, `When`, `Billing`) is explained in [../model-catalog/AGENTS.md](../model-catalog/AGENTS.md#concepts), the only crate that maps it to JSON keys

litellm-rust/crates/cost/src/price.rs

@@ -19,6 +19,7 @@ pub enum ServiceTier {
     Balanced,
 }
 
+/// A prompt-size range. `OverPromptTokens(n)` is spelled `_above_<n>_tokens` or `_above_<n / 1000>k_tokens`
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Band {
     #[default]
@@ -26,12 +27,15 @@ pub enum Band {
     OverPromptTokens(NonZeroU64),
 }
 
+/// The request condition a price applies under, its service tier and prompt band
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct When {
     pub tier: ServiceTier,
     pub band: Band,
 }
 
+/// The route whose price list a call reads, from the `_batches` and `_decisions` suffixes. Not part of `When`,
+/// since a missing batch price is half the online rate and a missing decisions price is the full online rate
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Channel {
     #[default]
@@ -40,6 +44,7 @@ pub enum Channel {
     Decisions,
 }
 
+/// The key of one rate, what is counted, through which channel and under which condition
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Price {
     pub meter: Meter,
@@ -47,6 +52,7 @@ pub struct Price {
     pub when: When,
 }
 
+/// Every rate a row lists, its off-peak overlay and the row attributes that say how the rates apply
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct Prices {
     rates: BTreeMap<Price, UnitPrice>,
@@ -117,6 +123,7 @@ pub enum TierScope {
     WholeCall,
 }
 
+/// Rates that replace the row's token rates while a UTC window in the schedule is open
 #[derive(Clone, Debug, PartialEq)]
 pub struct OffPeak {
     schedule: Schedule,
@@ -133,6 +140,7 @@ pub struct OffPeakRates {
     pub reasoning: Option<UnitPrice>,
 }
 
+/// Multipliers applied on top of a priced call, never prices themselves
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct Adjustments(BTreeMap<Uplift, Multiplier>);
 

litellm-rust/crates/cost/src/price/meter.rs

@@ -1,5 +1,6 @@
 use std::num::NonZeroU64;
 
+/// What a price counts. A variation only one meter has, such as cache TTL or image size, lives inside its variant
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Meter {
     Input(InputUnit),
@@ -51,12 +52,14 @@ pub enum OutputUnit {
     VideoSecond,
 }
 
+/// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CacheReadMode {
     Implicit,
     Explicit,
 }
 
+/// How long a cache write lives. `OneHour` and `TwelveHours` are spelled `_above_1hr` and `_above_12hr`
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CacheTtl {
     Default,

litellm-rust/crates/host-http/tests/serve.rs

@@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(Bytes::from(wire.url)))
         },
@@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once(
         move |_, _, route_hooks, _observations| async move {
             if event {
                 route_hooks
-                    .after_provider_response(RawResponse {
-                        body: "response".into(),
-                    })
+                    .after_provider_response(RawResponse::new("response".into()))
                     .await?;
             } else {
                 route_hooks
@@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(json!({"url": wire.url})))
         },
@@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting(
         None,
         |_, _, route_hooks, _observations| async move {
             route_hooks
-                .after_provider_response(RawResponse { body: "raw".into() })
+                .after_provider_response(RawResponse::new("raw".into()))
                 .await?;
             Err(TestError::Provider)
         },

litellm-rust/crates/host-native/tests/driver.rs

@@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine {
                 )
                 .await?;
             route_hooks
-                .after_provider_response(RawResponse {
-                    body: wire.url.clone(),
-                })
+                .after_provider_response(RawResponse::new(wire.url.clone()))
                 .await?;
             Ok(CallOutput::Complete(wire.url))
         },
@@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re
         "input",
         Some(sender),
         |_, _, interceptors, observers| async move {
-            let raw = RawResponse {
-                body: "provider response".into(),
-            };
+            let raw = RawResponse::new("provider response".into());
             observers.unwrap().emit(CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
             ));

litellm-rust/crates/host-python/tests/support/mod.rs

@@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory {
                     .before_provider_request(wire(), context())
                     .await?;
                 host.interceptors
-                    .after_provider_response(RawResponse { body: "raw".into() })
+                    .after_provider_response(RawResponse::new("raw".into()))
                     .await?;
                 Ok(format!("{projected}|{signed}|{}", wire.url))
             })

litellm-rust/crates/host/src/interceptors.rs

@@ -28,6 +28,28 @@ pub struct RequestContext {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct RawResponse {
     pub body: String,
+    /// The provider's HTTP status and headers, when the route forwards them to logging.
+    pub head: Option<ResponseHead>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ResponseHead {
+    pub status: u16,
+    pub headers: Vec<(String, String)>,
+}
+
+impl RawResponse {
+    pub fn new(body: String) -> Self {
+        Self { body, head: None }
+    }
+
+    #[must_use]
+    pub fn with_head(self, head: ResponseHead) -> Self {
+        Self {
+            head: Some(head),
+            ..self
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -186,7 +208,7 @@ mod tests {
                 .await?;
                 Interceptors::after_provider_response(
                     &channel.interceptors,
-                    RawResponse { body: "raw".into() },
+                    RawResponse::new("raw".into()),
                 )
                 .await?;
                 Ok((sent, ()))

litellm-rust/crates/host/tests/observation.rs

@@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile
 
 #[rstest]
 fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) {
-    let mut raw = RawResponse {
-        body: "provider response".into(),
-    };
+    let mut raw = RawResponse::new("provider response".into());
     let event: CallEvent<(), (), &RawResponse> =
         CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw });
     let expected = raw.clone();

litellm-rust/crates/http/src/error.rs

@@ -58,8 +58,20 @@ pub(crate) enum Repr {
     Network(#[source] reqwest::Error),
     #[error("upstream request timed out")]
     Timeout(#[source] reqwest::Error),
+    /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says
+    /// the response head had already arrived.
+    #[error("upstream request timed out")]
+    ReadTimeout {
+        #[source]
+        elapsed: Elapsed,
+        body: bool,
+    },
     #[error("upstream request failed with status {status}: {body}")]
-    Status { status: u16, body: UpstreamBody },
+    Status {
+        status: u16,
+        body: UpstreamBody,
+        headers: Vec<(String, String)>,
+    },
 
     #[error("invalid WebSocket URL")]
     WebSocketUrl(#[source] url::ParseError),
@@ -136,13 +148,59 @@ impl Repr {
 
 impl Error {
     pub fn upstream(status: u16, body: String) -> Self {
+        Self::upstream_with_headers(status, body, Vec::new())
+    }
+
+    /// An upstream failure that keeps the provider's response headers (`retry-after`,
+    /// `request-id`) for the caller.
+    pub fn upstream_with_headers(
+        status: u16,
+        body: String,
+        headers: Vec<(String, String)>,
+    ) -> Self {
         Repr::Status {
             status,
             body: UpstreamBody(body),
+            headers,
         }
         .into()
     }
 
+    /// The provider's response headers on an upstream failure; empty for every other error.
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match &*self.0 {
+            Repr::Status { headers, .. } => headers,
+            _ => &[],
+        }
+    }
+
+    /// A timeout reached before the connection was established, as opposed to while reading.
+    pub fn is_connect_timeout(&self) -> bool {
+        matches!(&*self.0, Repr::Timeout(error) if error.is_connect())
+    }
+
+    /// A read the caller timed itself (httpx's per-read timeout) ran out.
+    pub fn read_timeout(elapsed: Elapsed) -> Self {
+        Repr::ReadTimeout {
+            elapsed,
+            body: false,
+        }
+        .into()
+    }
+
+    /// A read of the response body the caller timed itself ran out, after the head arrived.
+    pub fn body_read_timeout(elapsed: Elapsed) -> Self {
+        Repr::ReadTimeout {
+            elapsed,
+            body: true,
+        }
+        .into()
+    }
+
+    pub fn is_body_read_timeout(&self) -> bool {
+        matches!(&*self.0, Repr::ReadTimeout { body: true, .. })
+    }
+
     pub fn computed_header(name: impl Into<String>) -> Self {
         Repr::ComputedHeader(name.into()).into()
     }
@@ -169,6 +227,7 @@ impl Error {
         match &*self.0 {
             Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::Handshake(_)
             | Repr::HandshakeTimeout(_)
@@ -227,6 +286,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::Handshake(_)
             | Repr::HandshakeTimeout(_)
@@ -259,6 +319,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -295,6 +356,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -333,6 +395,7 @@ impl Error {
             | Repr::Signature(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
@@ -356,7 +419,10 @@ impl Error {
 
     pub fn is_timeout(&self) -> bool {
         match &*self.0 {
-            Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true,
+            Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
+            | Repr::HandshakeTimeout(_)
+            | Repr::DownloadTimeout(_) => true,
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -392,7 +458,7 @@ impl Error {
 
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match &*self.0 {
-            Repr::Status { status, body } => Some((*status, &body.0)),
+            Repr::Status { status, body, .. } => Some((*status, &body.0)),
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -408,6 +474,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)
             | Repr::WebSocketSchemeRewrite(_)
@@ -431,7 +498,7 @@ impl Error {
     pub fn status(&self) -> Option<u16> {
         match &*self.0 {
             Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status),
-            Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408),
+            Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408),
             Repr::Read { .. }
             | Repr::InvalidPem { .. }
             | Repr::Client(_)
@@ -482,6 +549,7 @@ impl Error {
             | Repr::Connect(_)
             | Repr::Network(_)
             | Repr::Timeout(_)
+            | Repr::ReadTimeout { .. }
             | Repr::Status { .. }
             | Repr::WebSocketUrl(_)
             | Repr::WebSocketScheme(_)

litellm-rust/crates/inference-chat/src/handler.rs

@@ -108,7 +108,7 @@ pub(super) async fn execute(
             if !status.is_success() {
                 return Err(litellm_http::Error::upstream(status.as_u16(), text).into());
             }
-            let raw = RawResponse { body: text.clone() };
+            let raw = RawResponse::new(text.clone());
             if let Some(observers) = observers {
                 observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                     ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference-messages/Cargo.toml

@@ -24,6 +24,7 @@ litellm-tracing.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
+tokio.workspace = true
 tracing.workspace = true
 
 [dev-dependencies]
@@ -34,5 +35,4 @@ litellm-host-native.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }
 litellm-inference-testing.workspace = true
 rstest.workspace = true
-tokio.workspace = true
 wiremock.workspace = true

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -35,6 +35,14 @@ impl MessagesProvider {
     pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
         self.config
     }
+
+    /// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped.
+    pub(crate) fn accepts_service_tier(self) -> bool {
+        matches!(
+            self.provider,
+            LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle
+        )
+    }
 }
 
 /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
@@ -119,3 +127,69 @@ mod tests {
         );
     }
 }
+
+/// Python's `_merge_messages_header_sources`: later sources win per header name, except
+/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and
+/// sorted into one lowercase `anthropic-beta` entry.
+pub fn merge_messages_header_sources(
+    sources: impl IntoIterator<Item = Map<String, Value>>,
+) -> Map<String, Value> {
+    let mut merged = Map::new();
+    let mut betas = std::collections::BTreeSet::new();
+    for (name, value) in sources.into_iter().flatten() {
+        if !name.eq_ignore_ascii_case("anthropic-beta") {
+            merged.insert(name, value);
+            continue;
+        }
+        let values: Vec<&str> = match &value {
+            Value::String(text) => vec![text.as_str()],
+            Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(),
+            _ => Vec::new(),
+        };
+        betas.extend(
+            values
+                .iter()
+                .flat_map(|value| value.split(','))
+                .map(str::trim)
+                .filter(|beta| !beta.is_empty())
+                .map(str::to_owned),
+        );
+    }
+    if !betas.is_empty() {
+        merged.insert(
+            "anthropic-beta".to_owned(),
+            Value::String(betas.into_iter().collect::<Vec<_>>().join(",")),
+        );
+    }
+    merged
+}
+
+#[cfg(test)]
+mod merge_tests {
+    use serde_json::{Map, Value, json};
+
+    use super::merge_messages_header_sources;
+
+    fn map(value: Value) -> Map<String, Value> {
+        value.as_object().cloned().unwrap()
+    }
+
+    #[test]
+    fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() {
+        let merged = merge_messages_header_sources([
+            map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})),
+            map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})),
+            map(json!({"anthropic-beta": 5})),
+        ]);
+        assert_eq!(
+            Value::Object(merged),
+            json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"})
+        );
+    }
+
+    #[test]
+    fn no_beta_values_leave_no_beta_header() {
+        let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]);
+        assert!(merged.is_empty());
+    }
+}

litellm-rust/crates/inference-messages/src/handler.rs

@@ -10,9 +10,9 @@ use litellm_core_utils::{
     get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
 };
 use litellm_host::interceptors::{
-    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
+    ResponseHead, ResultSource, WireRequest,
 };
-use litellm_http::request::with_default_headers;
 use litellm_llms::base_llm::{
     auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
@@ -21,14 +21,16 @@ use litellm_llms::base_llm::{
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
     Error, MessagesCall, MessagesCallResponse, MessagesRoute,
-    common_utils::{MessagesProvider, string_headers},
+    common_utils::{MessagesProvider, merge_messages_header_sources, string_headers},
     constants::MESSAGES_TIMEOUT_SECS,
     route::Messages,
     types::invalid_request,
@@ -126,14 +128,18 @@ async fn provider_call(
         MessagesRequest { model, ..body },
         shaping.settings.reasoning_auto_summary,
     )?;
+    let transform_context =
+        MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params);
+    let filtered = requested_optional_params(sanitized, provider, &transform_context)?;
     let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+        without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?;
 
     let scoped =
         get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
+    let forwarded = string_headers(Some(merge_messages_header_sources([
+        extra_headers.unwrap_or_default(),
+        scoped,
+    ])))?;
     let validated = config.validate_environment(
         forwarded,
         api_key.as_deref(),
@@ -144,16 +150,13 @@ async fn provider_call(
     )?;
 
     let body = config.request_body(
-        config.transform_anthropic_messages_request(
-            trimmed,
-            &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-        )?,
+        config.transform_anthropic_messages_request(trimmed, &transform_context)?,
         &validated.headers,
         shaping.settings.drop_params,
     )?;
     let environment = ValidatedEnvironment {
         headers: config.request_headers(
-            with_default_headers(validated.headers, config.default_headers()),
+            with_python_default_headers(validated.headers, config.default_headers()),
             &body,
         ),
         auth: validated.auth,
@@ -184,7 +187,7 @@ async fn provider_call(
         .before_provider_request(
             WireRequest {
                 url,
-                headers: authenticated.headers,
+                headers: joined_header_values(authenticated.headers),
                 body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
             request_context,
@@ -213,6 +216,64 @@ async fn provider_call(
     })
 }
 
+/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in
+/// case into one lowercase header whose values are joined with `", "`.
+fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> {
+    let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len());
+    let mut positions: std::collections::HashMap<String, usize> =
+        std::collections::HashMap::with_capacity(headers.len());
+    for (name, value) in headers {
+        match positions.entry(name.to_ascii_lowercase()) {
+            std::collections::hash_map::Entry::Occupied(position) => {
+                let present = &mut joined[*position.get()].1;
+                present.push_str(", ");
+                present.push_str(&value);
+            }
+            std::collections::hash_map::Entry::Vacant(position) => {
+                position.insert(joined.len());
+                joined.push((name, value));
+            }
+        }
+    }
+    joined
+}
+
+/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is
+/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it.
+fn with_python_default_headers(
+    headers: Vec<(String, String)>,
+    defaults: &[(&str, &str)],
+) -> Vec<(String, String)> {
+    let missing: Vec<(String, String)> = defaults
+        .iter()
+        .filter(|(name, _)| !headers.iter().any(|(present, _)| present == name))
+        .map(|(name, value)| ((*name).to_string(), (*value).to_string()))
+        .collect();
+    headers.into_iter().chain(missing).collect()
+}
+
+/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier
+/// and sampling params before `additional_drop_params` and the environment check run.
+fn requested_optional_params(
+    request: MessagesRequest,
+    provider: MessagesProvider,
+    context: &MessagesTransformContext,
+) -> Result<MessagesRequest, Error> {
+    let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params(
+        request, context,
+    )?;
+    if provider.accepts_service_tier() {
+        return Ok(request);
+    }
+    Ok(MessagesRequest {
+        params: MessagesOptionalParams {
+            service_tier: None,
+            ..request.params
+        },
+        ..request
+    })
+}
+
 fn without_additional_drop_params(
     request: MessagesRequest,
     paths: &[String],
@@ -245,6 +306,7 @@ async fn call_provider(
     } = request;
     let provider_name = provider.as_str();
     log_request_body(provider_name, stream, &wire.body);
+    let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
     let response = send(
         http,
         Authenticated {
@@ -254,10 +316,11 @@ async fn call_provider(
         &wire.url,
         &wire.body,
         timeout,
+        stream,
     )
     .await?;
     if !response.status().is_success() {
-        return Err(provider_error(response).await);
+        return Err(provider_error(response, timeout).await);
     }
     let config = provider.config();
     if stream {
@@ -265,11 +328,24 @@ async fn call_provider(
             response,
             config.stream_decoder(),
             provider_name,
+            timeout,
         ));
     }
+    let head = ResponseHead {
+        status: response.status().as_u16(),
+        headers: litellm_http::request::header_pairs(response.headers()),
+    };
+    let status = head.status;
     let text = response.text().await.map_err(litellm_http::Error::from)?;
     log_response_body(&text);
-    context.response_received(&text).await?;
+    context
+        .raw_response_received(RawResponse::new(text.clone()).with_head(head))
+        .await?;
+    if serde_json::from_str::<serde::de::IgnoredAny>(&text).is_err() {
+        // Python raises `AnthropicError(status_code=<the response's>, message=<its text>)` for a
+        // body it cannot parse, and the public mapping classifies it by that status.
+        return Err(litellm_http::Error::upstream(status, text).into());
+    }
     decode_response(config, &identity.model, &text)
         .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
@@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error {
     ))
 }
 
+/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete
+/// response keeps the client's overall deadline; a stream is bounded until its head arrives and
+/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off.
 async fn send(
     http: &litellm_http::Client,
     authenticated: Authenticated,
     url: &str,
     body: &Value,
-    timeout: Option<Duration>,
+    timeout: Duration,
+    stream: bool,
 ) -> Result<reqwest::Response, Error> {
-    let request = outbound_request(
-        authenticated,
-        url,
-        body,
-        Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))),
-    )?;
-    Ok(litellm_inference::outbound::send(request, http).await?)
+    let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?;
+    if !stream {
+        return Ok(litellm_inference::outbound::send(request, http).await?);
+    }
+    tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http))
+        .await
+        .map_err(litellm_http::Error::read_timeout)?
+        .map_err(Error::from)
 }
 
-async fn provider_error(response: reqwest::Response) -> Error {
+/// The stream's body, each read bounded by `timeout` as httpx bounds each read.
+fn timed_chunks(
+    response: reqwest::Response,
+    timeout: Duration,
+) -> impl futures_util::Stream<Item = Result<Bytes, litellm_http::Error>> + Send + 'static {
+    futures_util::stream::try_unfold(response, move |mut response| async move {
+        let chunk = tokio::time::timeout(timeout, response.chunk())
+            .await
+            .map_err(litellm_http::Error::body_read_timeout)?
+            .map_err(litellm_http::Error::from)?;
+        Ok(chunk.map(|chunk| (chunk, response)))
+    })
+}
+
+/// The provider's error, its body read under `timeout`: a stream has no overall deadline once
+/// its head arrives, so an error body that stalls would otherwise hold the call forever.
+async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error {
     let status = response.status().as_u16();
-    match response.text().await {
-        Ok(text) => {
+    let headers = litellm_http::request::header_pairs(response.headers());
+    match tokio::time::timeout(timeout, response.text()).await {
+        Ok(Ok(text)) => {
             log_error_body(status, &text);
-            litellm_http::Error::upstream(status, text).into()
+            litellm_http::Error::upstream_with_headers(status, text, headers).into()
         }
-        Err(error) => litellm_http::Error::from(error).into(),
+        Ok(Err(error)) => litellm_http::Error::from(error).into(),
+        Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(),
     }
 }
 
@@ -329,22 +428,22 @@ fn streaming_response(
     response: reqwest::Response,
     decoder: Option<StreamDecoder>,
     provider: &'static str,
+    timeout: Duration,
 ) -> MessagesCallResponse {
     let headers = response
         .headers()
         .iter()
         .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string())))
         .collect();
+    let chunks = timed_chunks(response, timeout)
+        .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
     let chunks = match decoder {
-        None => futures_util::stream::try_unfold(response, move |mut response| async move {
-            let chunk = response.chunk().await.map_err(litellm_http::Error::from)?;
-            Ok(chunk.map(|chunk| {
-                log_chunk(provider, "provider_response", &chunk);
-                (chunk, response)
-            }))
-        })
-        .boxed(),
-        Some(decode) => decoded_chunks(response, decode, provider),
+        None => chunks.map_err(Error::from).boxed(),
+        Some(decode) => decoded_chunks(
+            chunks.map_err(std::io::Error::other).boxed(),
+            decode,
+            provider,
+        ),
     };
     MessagesCallResponse::Stream {
         head: super::route::MessagesStreamHead { headers },
@@ -353,15 +452,10 @@ fn streaming_response(
 }
 
 fn decoded_chunks(
-    response: reqwest::Response,
+    bytes: ByteStream,
     decode: StreamDecoder,
     provider: &'static str,
 ) -> BoxStream<'static, Result<Bytes, Error>> {
-    let bytes: ByteStream = response
-        .bytes_stream()
-        .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk))
-        .map_err(|error| std::io::Error::other(litellm_http::Error::from(error)))
-        .boxed();
     futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
         let Some(event) = events.try_next().await? else {
             return Ok(None);
@@ -964,9 +1058,12 @@ mod tests {
             .send()
             .await
             .unwrap();
-        let MessagesCallResponse::Stream { mut chunks, .. } =
-            streaming_response(response, Some(streaming::event_stream), "test")
-        else {
+        let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
+            response,
+            Some(streaming::event_stream),
+            "test",
+            Duration::from_secs(5),
+        ) else {
             panic!("a streaming response returns chunks");
         };
 

litellm-rust/crates/inference-messages/src/lib.rs

@@ -14,6 +14,7 @@ use std::sync::Arc;
 
 use crate::common_utils::messages_provider;
 
+pub use common_utils::merge_messages_header_sources;
 pub use litellm_inference::Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
 #[case::deepseek("deepseek")]
 #[case::minimax("minimax")]
 #[tokio::test]
-async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
+/// Python adds a default unless that exact lowercase name is present, then folds the casings
+/// into one header, so a capitalised caller header is sent beside the default.
+async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults(
+    call: MessagesCall,
+    #[case] provider: &str,
+) {
     let upstream = upstream([message_response()]).await;
 
     run_message(MessagesCall {
@@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
     .await;
 
     let request = only_request(&upstream).await;
-    assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]);
+    assert_eq!(
+        request.header_values("anthropic-version"),
+        ["2024-01-01, 2023-06-01"]
+    );
     assert_eq!(
         request.header_values("content-type"),
-        ["application/json; charset=utf-8"]
+        ["application/json; charset=utf-8, application/json"]
     );
 }
 

litellm-rust/crates/inference-messages/tests/messages/response.rs

@@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case]
 }
 
 #[rstest]
-#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))]
-#[case::not_a_message(json_response(json!({"unexpected": true})))]
 #[tokio::test]
-async fn an_unreadable_success_body_is_an_invalid_response(
-    call: MessagesCall,
-    #[case] response: ResponseTemplate,
-) {
-    let upstream = upstream([response]).await;
+async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) {
+    let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await;
+
+    let error = run(MessagesCall {
+        api_key: Some("sk".into()),
+        api_base: Some(upstream.uri()),
+        ..call
+    })
+    .await
+    .expect_err("an unparseable body fails");
+
+    assert_eq!(error.upstream_response(), Some((200, "not json")));
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) {
+    let upstream = upstream([json_response(json!({"unexpected": true}))]).await;
 
     let error = run(MessagesCall {
         api_key: Some("sk".into()),

litellm-rust/crates/inference-messages/tests/messages/stream.rs

@@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream(
         .unwrap();
 }
 
+/// Answers with an error head whose promised body never arrives.
+async fn stalling_error_upstream() -> (String, JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let base = format!("http://{}", listener.local_addr().unwrap());
+    let connection = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.unwrap();
+        let mut request = vec![0; 4096];
+        let _ = socket.read(&mut request).await;
+        socket
+            .write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n")
+            .await
+            .unwrap();
+        let _ = socket.read_to_end(&mut Vec::new()).await;
+    });
+    (base, connection)
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) {
+    let (base, connection) = stalling_error_upstream().await;
+    let host = RecordingStreamHost::new(
+        MessagesCall {
+            timeout: Some(Duration::from_millis(300)),
+            ..streaming(call, base)
+        },
+        usize::MAX,
+    );
+
+    let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host))
+        .await
+        .expect("the stalled error body gives up within the timeout")
+        .expect_err("a stalled error body fails the call");
+
+    assert!(error.dispatched(), "{error:?}");
+    assert_eq!(error.upstream_response(), None);
+    assert!(host.seen.into_inner().unwrap().is_empty());
+    tokio::time::timeout(Duration::from_secs(5), connection)
+        .await
+        .expect("timing out closes the upstream connection")
+        .unwrap();
+}
+
 #[rstest]
 #[tokio::test]
 async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) {

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -91,9 +91,7 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
     }
 
     fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-        let raw = RawResponse {
-            body: String::from_utf8_lossy(body).into_owned(),
-        };
+        let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned());
         if let Some(observers) = self.observers {
             observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference-responses/src/handler.rs

@@ -114,7 +114,7 @@ pub(super) async fn execute(
                 });
             }
             let body = response.text().await.map_err(litellm_http::Error::from)?;
-            let raw = RawResponse { body: body.clone() };
+            let raw = RawResponse::new(body.clone());
             if let Some(observers) = observers {
                 observers.emit(litellm_host::lifecycle::CallEvent::Execution(
                     ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference/src/context.rs

@@ -42,9 +42,11 @@ impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
     }
 
     pub async fn response_received(&self, body: &str) -> Result<(), Error> {
-        let raw = RawResponse {
-            body: body.to_owned(),
-        };
+        self.raw_response_received(RawResponse::new(body.to_owned()))
+            .await
+    }
+
+    pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> {
         if let Some(observers) = &self.observers {
             observers.emit(CallEvent::Execution(
                 ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

litellm-rust/crates/inference/src/error.rs

@@ -146,6 +146,34 @@ impl Error {
         }
     }
 
+    /// The provider's response headers on an upstream failure; empty for every other error.
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match self {
+            Self::Llm(error) => error.upstream_headers(),
+            Self::Http(error) => error.upstream_headers(),
+            _ => &[],
+        }
+    }
+
+    /// A read of a streamed body timed out after the response head arrived.
+    pub fn body_read_timeout(&self) -> bool {
+        match self {
+            Self::Llm(error) => error.is_body_read_timeout(),
+            Self::Http(error) => error.is_body_read_timeout(),
+            _ => false,
+        }
+    }
+
+    /// `Some(connect)` when the call timed out, where `connect` says the connection was never
+    /// established; `None` for every other error.
+    pub fn timeout(&self) -> Option<bool> {
+        match self {
+            Self::Llm(error) => error.is_timeout(),
+            Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+            _ => None,
+        }
+    }
+
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match self {
             Self::Llm(error) => error.upstream_response(),

litellm-rust/crates/llms-types/src/formats/messages/request.rs

@@ -264,7 +264,12 @@ pub enum ThinkingDisplay {
 #[macro_rules_attribute::apply(crate::wire_type)]
 #[derive(Default)]
 pub struct EnabledThinking {
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    /// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0.
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "crate::recognized::present"
+    )]
     pub budget_tokens: Option<Recognized<u64>>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub display: Option<Recognized<ThinkingDisplay>>,

litellm-rust/crates/llms-types/src/formats/messages/response.rs

@@ -10,14 +10,29 @@ pub struct MessagesResponse {
     pub content: Vec<Value>,
     pub stop_reason: Option<String>,
     pub stop_sequence: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    /// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps.
+    #[serde(
+        default,
+        deserialize_with = "present_value",
+        skip_serializing_if = "Option::is_none"
+    )]
     pub usage: Option<Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(
+        default,
+        deserialize_with = "present_value",
+        skip_serializing_if = "Option::is_none"
+    )]
     pub container: Option<Value>,
     #[serde(flatten)]
     pub extra: Map<String, Value>,
 }
 
+fn present_value<'de, D: serde::Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<Value>, D::Error> {
+    <Value as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -73,4 +88,21 @@ mod tests {
         assert_eq!(body.get("usage").cloned(), usage);
         assert_eq!(body.get("container").cloned(), container);
     }
+
+    #[test]
+    fn explicit_null_usage_and_container_survive_a_round_trip() {
+        let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
+            "content": [], "stop_reason": null, "stop_sequence": null,
+            "usage": null, "container": null});
+        let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
+        assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
+    }
+
+    #[test]
+    fn absent_usage_and_container_stay_absent() {
+        let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
+            "content": [], "stop_reason": null, "stop_sequence": null});
+        let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
+        assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
+    }
 }

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -124,12 +124,86 @@ pub enum AnthropicBeta {
     ToolSearchTool20251019,
     #[strum(serialize = "inline-tools-2026-09-15")]
     InlineTools20260915,
+    #[strum(serialize = "afk-mode-2026-01-31")]
+    AfkMode20260131,
+    #[strum(serialize = "agent-memory-2026-07-22")]
+    AgentMemory20260722,
+    #[strum(serialize = "analytics-usage-cost-2026-08-24")]
+    AnalyticsUsageCost20260824,
+    #[strum(serialize = "cache-diagnosis-2026-04-07")]
+    CacheDiagnosis20260407,
+    #[strum(serialize = "ce-plugins-2026-09-01")]
+    CePlugins20260901,
+    #[strum(serialize = "ce-user-management-2026-07-13")]
+    CeUserManagement20260713,
+    #[strum(serialize = "code-execution-2025-05-22")]
+    CodeExecution20250522,
+    #[strum(serialize = "computer-use-2024-10-22")]
+    ComputerUse20241022,
+    #[strum(serialize = "dev-full-thinking-2025-05-14")]
+    DevFullThinking20250514,
+    #[strum(serialize = "dreaming-2026-04-21")]
+    Dreaming20260421,
+    #[strum(serialize = "experimental-claude-tag-2026-10-21")]
+    ExperimentalClaudeTag20261021,
+    #[strum(serialize = "extended-cache-ttl-2025-04-11")]
+    ExtendedCacheTtl20250411,
+    #[strum(serialize = "fallback-credit-2026-06-01")]
+    FallbackCredit20260601,
+    #[strum(serialize = "fallback-credit-2026-07-01")]
+    FallbackCredit20260701,
+    #[strum(serialize = "managed-agents-2026-04-01")]
+    ManagedAgents20260401,
+    #[strum(serialize = "mcp-client-2026-09-15")]
+    McpClient20260915,
+    #[strum(serialize = "mcp-tunnels-2026-06-22")]
+    McpTunnels20260622,
+    #[strum(serialize = "message-batches-2024-09-24")]
+    MessageBatches20240924,
+    #[strum(serialize = "message-threads-2026-08-12")]
+    MessageThreads20260812,
+    #[strum(serialize = "mid-conversation-system-clear-at-2026-08-21")]
+    MidConversationSystemClearAt20260821,
+    #[strum(serialize = "model-context-window-exceeded-2025-08-26")]
+    ModelContextWindowExceeded20250826,
+    #[strum(serialize = "output-300k-2026-03-24")]
+    Output300k20260324,
+    #[strum(serialize = "pdfs-2024-09-25")]
+    Pdfs20240925,
+    #[strum(serialize = "prompt-caching-2024-07-31")]
+    PromptCaching20240731,
+    #[strum(serialize = "redact-thinking-2026-02-12")]
+    RedactThinking20260212,
+    #[strum(serialize = "server-side-fallback-2026-06-01")]
+    ServerSideFallback20260601,
+    #[strum(serialize = "server-side-fallback-2026-07-01")]
+    ServerSideFallback20260701,
+    #[strum(serialize = "spend-limit-reads-2026-09-26")]
+    SpendLimitReads20260926,
+    #[strum(serialize = "structured-outputs-2025-12-15")]
+    StructuredOutputs20251215,
+    #[strum(serialize = "task-budgets-2026-03-13")]
+    TaskBudgets20260313,
+    #[strum(serialize = "telemetry-destinations-2026-08-11")]
+    TelemetryDestinations20260811,
+    #[strum(serialize = "thinking-token-count-2026-05-13")]
+    ThinkingTokenCount20260513,
+    #[strum(serialize = "token-counting-2024-11-01")]
+    TokenCounting20241101,
+    #[strum(serialize = "user-profiles-2026-03-24")]
+    UserProfiles20260324,
+    #[strum(serialize = "user-profiles-2026-08-18")]
+    UserProfiles20260818,
+    #[strum(serialize = "user-profiles-2026-09-04")]
+    UserProfiles20260904,
+    #[strum(serialize = "webhooks-2026-10-07")]
+    Webhooks20261007,
     #[strum(default, transparent)]
     Other(String),
 }
 
 impl AnthropicBeta {
-    pub const KNOWN: [Self; 40] = [
+    pub const KNOWN: [Self; 77] = [
         Self::Oauth20250420,
         Self::WebFetch20250910,
         Self::WebSearch20250305,
@@ -170,6 +244,43 @@ impl AnthropicBeta {
         Self::ToolExamples20251029,
         Self::ToolSearchTool20251019,
         Self::InlineTools20260915,
+        Self::AfkMode20260131,
+        Self::AgentMemory20260722,
+        Self::AnalyticsUsageCost20260824,
+        Self::CacheDiagnosis20260407,
+        Self::CePlugins20260901,
+        Self::CeUserManagement20260713,
+        Self::CodeExecution20250522,
+        Self::ComputerUse20241022,
+        Self::DevFullThinking20250514,
+        Self::Dreaming20260421,
+        Self::ExperimentalClaudeTag20261021,
+        Self::ExtendedCacheTtl20250411,
+        Self::FallbackCredit20260601,
+        Self::FallbackCredit20260701,
+        Self::ManagedAgents20260401,
+        Self::McpClient20260915,
+        Self::McpTunnels20260622,
+        Self::MessageBatches20240924,
+        Self::MessageThreads20260812,
+        Self::MidConversationSystemClearAt20260821,
+        Self::ModelContextWindowExceeded20250826,
+        Self::Output300k20260324,
+        Self::Pdfs20240925,
+        Self::PromptCaching20240731,
+        Self::RedactThinking20260212,
+        Self::ServerSideFallback20260601,
+        Self::ServerSideFallback20260701,
+        Self::SpendLimitReads20260926,
+        Self::StructuredOutputs20251215,
+        Self::TaskBudgets20260313,
+        Self::TelemetryDestinations20260811,
+        Self::ThinkingTokenCount20260513,
+        Self::TokenCounting20241101,
+        Self::UserProfiles20260324,
+        Self::UserProfiles20260818,
+        Self::UserProfiles20260904,
+        Self::Webhooks20261007,
     ];
 
     pub fn as_str(&self) -> &str {
@@ -195,10 +306,9 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 => {
+            Self::AdvancedToolUse20251120 | Self::ToolSearchTool20251019 => {
                 &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
-            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -207,98 +317,99 @@ impl AnthropicBeta {
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260904 => &[
+            Self::AfkMode20260131
+            | Self::CePlugins20260901
+            | Self::CeUserManagement20260713
+            | Self::Compact20260904
+            | Self::Dreaming20260421
+            | Self::ExperimentalClaudeTag20261021
+            | Self::ExtendedCacheTtl20250411
+            | Self::FallbackCredit20260601
+            | Self::FallbackCredit20260701
+            | Self::InlineTools20260915
+            | Self::ManagedAgents20260401
+            | Self::McpTunnels20260622
+            | Self::MidConversationSystemClearAt20260821
+            | Self::ModelContextWindowExceeded20250826
+            | Self::Oauth20250420
+            | Self::Pdfs20240925
+            | Self::PromptCaching20240731
+            | Self::PromptCachingScope20260105
+            | Self::RedactThinking20260212
+            | Self::ServerSideFallback20260601
+            | Self::SpendLimitReads20260926
+            | Self::StructuredOutputs20251215
+            | Self::TaskBudgets20260313
+            | Self::TelemetryDestinations20260811
+            | Self::ThinkingTokenCount20260513
+            | Self::TokenCounting20241101
+            | Self::UserProfiles20260324
+            | Self::UserProfiles20260818
+            | Self::UserProfiles20260904 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Bash20241022
+            Self::AgentMemory20260722
+            | Self::Bash20241022
             | Self::Bash20250124
+            | Self::DevFullThinking20250514
             | Self::McpServers20251204
             | Self::StructuredOutput20240301
             | Self::TextEditor20241022
             | Self::TextEditor20250124 => BetaProvider::VARIANTS,
-            Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::CodeExecution20250825
+            Self::AnalyticsUsageCost20260824
+            | Self::CacheDiagnosis20260407
+            | Self::CodeExecution20250522
+            | Self::CodeExecution20250825
+            | Self::ComputerUse20241022
+            | Self::FastMode20260201
             | Self::FilesApi20250414
             | Self::McpClient20250404
             | Self::McpClient20251120
-            | Self::PromptCachingScope20260105
+            | Self::McpClient20260915
+            | Self::MessageBatches20240924
+            | Self::MessageThreads20260812
+            | Self::Output300k20260324
+            | Self::ServerSideFallback20260701
             | Self::Skills20251002
-            | Self::WebFetch20250910 => &[
+            | Self::WebFetch20250910
+            | Self::Webhooks20261007 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
+            Self::ClaudeCode20250219
+            | Self::PerTurnControl20260701
+            | Self::ToolExamples20251029 => &[
+                BetaProvider::BedrockConverse,
+                BetaProvider::Bedrock,
+                BetaProvider::Databricks,
+            ],
+            Self::Compact20260112
+            | Self::ContextManagement20250627
+            | Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
+            | Self::Effort20251124
+            | Self::FineGrainedToolStreaming20250514
             | Self::ThinkingBindingControls20260801
             | Self::ThinkingDisplayUpdates20260818 => &[],
-            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
-            Self::Effort20251124 => &[BetaProvider::VertexAi],
-            Self::FastMode20260201 | Self::Oauth20250420 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::FineGrainedToolStreaming20250514 => {
-                &[BetaProvider::AzureAi, BetaProvider::VertexAi]
-            }
-            Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
-                &[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
-            }
-            Self::MidConversationOutputConfig20260701 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::MidConversationToolChanges20260701 => &[
+            Self::InterleavedThinking20250514
+            | Self::Output128k20250219
+            | Self::TokenEfficientTools20250219
+            | Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock],
+            Self::MidConversationOutputConfig20260701
+            | Self::MidConversationToolChanges20260701 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-            ],
-            Self::PerTurnControl20260701 => &[
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::Databricks,
-            ],
-            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ToolSearchTool20251019 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Databricks,
-            ],
-            Self::InlineTools20260915 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::Databricks,
-            ],
+            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock],
             Self::Other(_) => BetaProvider::VARIANTS,
         }
     }
@@ -451,7 +562,44 @@ mod tests {
             AnthropicBeta::TokenEfficientTools20250219,
             AnthropicBeta::ToolExamples20251029,
             AnthropicBeta::ToolSearchTool20251019,
-            AnthropicBeta::InlineTools20260915
+            AnthropicBeta::InlineTools20260915,
+            AnthropicBeta::AfkMode20260131,
+            AnthropicBeta::AgentMemory20260722,
+            AnthropicBeta::AnalyticsUsageCost20260824,
+            AnthropicBeta::CacheDiagnosis20260407,
+            AnthropicBeta::CePlugins20260901,
+            AnthropicBeta::CeUserManagement20260713,
+            AnthropicBeta::CodeExecution20250522,
+            AnthropicBeta::ComputerUse20241022,
+            AnthropicBeta::DevFullThinking20250514,
+            AnthropicBeta::Dreaming20260421,
+            AnthropicBeta::ExperimentalClaudeTag20261021,
+            AnthropicBeta::ExtendedCacheTtl20250411,
+            AnthropicBeta::FallbackCredit20260601,
+            AnthropicBeta::FallbackCredit20260701,
+            AnthropicBeta::ManagedAgents20260401,
+            AnthropicBeta::McpClient20260915,
+            AnthropicBeta::McpTunnels20260622,
+            AnthropicBeta::MessageBatches20240924,
+            AnthropicBeta::MessageThreads20260812,
+            AnthropicBeta::MidConversationSystemClearAt20260821,
+            AnthropicBeta::ModelContextWindowExceeded20250826,
+            AnthropicBeta::Output300k20260324,
+            AnthropicBeta::Pdfs20240925,
+            AnthropicBeta::PromptCaching20240731,
+            AnthropicBeta::RedactThinking20260212,
+            AnthropicBeta::ServerSideFallback20260601,
+            AnthropicBeta::ServerSideFallback20260701,
+            AnthropicBeta::SpendLimitReads20260926,
+            AnthropicBeta::StructuredOutputs20251215,
+            AnthropicBeta::TaskBudgets20260313,
+            AnthropicBeta::TelemetryDestinations20260811,
+            AnthropicBeta::ThinkingTokenCount20260513,
+            AnthropicBeta::TokenCounting20241101,
+            AnthropicBeta::UserProfiles20260324,
+            AnthropicBeta::UserProfiles20260818,
+            AnthropicBeta::UserProfiles20260904,
+            AnthropicBeta::Webhooks20261007
         )]
         beta: AnthropicBeta,
     ) {
@@ -551,7 +699,7 @@ mod tests {
         BetaProvider::Anthropic,
         Some(AnthropicBeta::Oauth20250420)
     )]
-    #[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
+    #[case::rejected("skills-2025-10-02", BetaProvider::VertexAi, None)]
     #[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
     fn on_resolves_known_spellings_held_as_other(
         #[case] raw: &str,
@@ -580,12 +728,11 @@ mod tests {
 
     #[test]
     fn unknown_values_are_kept_verbatim() {
-        let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();
-        assert_eq!(
-            parsed,
-            AnthropicBeta::Other("claude-code-20250219".to_string())
+        let parsed: AnthropicBeta = "example-beta-2099-01-01".parse().unwrap();
+        assert!(
+            matches!(&parsed, AnthropicBeta::Other(value) if value == "example-beta-2099-01-01")
         );
-        assert_eq!(parsed.to_string(), "claude-code-20250219");
+        assert_eq!(parsed.to_string(), "example-beta-2099-01-01");
     }
 
     #[test]

litellm-rust/crates/llms-types/src/recognized.rs

@@ -7,6 +7,16 @@ pub enum Recognized<T> {
     Unrecognized(Value),
 }
 
+/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay
+/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`.
+pub fn present<'de, D, T>(deserializer: D) -> Result<Option<Recognized<T>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    Recognized<T>: serde::Deserialize<'de>,
+{
+    <Recognized<T> as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
+}
+
 impl<T> Recognized<T> {
     pub fn known(&self) -> Option<&T> {
         match self {

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -96,7 +96,10 @@ mod tests {
     }
 
     #[rstest]
-    #[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
+    #[case::azure_ai(
+        BetaProvider::AzureAi,
+        "advisor-tool-2026-03-01,example-beta-2099-01-01"
+    )]
     #[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
     #[case::vertex_ai(BetaProvider::VertexAi, " , ")]
     fn filter_removes_header_when_nothing_survives(

litellm-rust/crates/llms/src/anthropic/common_utils.rs

@@ -154,11 +154,42 @@ pub fn complete_anthropic_url(
 ) -> String {
     let api_base = resolve_anthropic_api_base(api_base, env_lookup);
 
-    let api_base = api_base.trim_end_matches('/');
-    if api_base.ends_with(MESSAGES_PATH) {
+    ensure_api_route(&api_base, MESSAGES_PATH)
+}
+
+/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds.
+fn url_path(url: &str) -> &str {
+    let before_fragment = url.split('#').next().unwrap_or_default();
+    let before_query = before_fragment.split('?').next().unwrap_or_default();
+    match before_query.split_once("://") {
+        Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]),
+        None => before_query,
+    }
+}
+
+/// Python's `_strip_trailing_slashes_keeping_the_scheme`.
+fn strip_trailing_slashes(api_base: &str) -> String {
+    let stripped = api_base.trim_end_matches('/');
+    if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') {
         return api_base.to_string();
     }
-    format!("{api_base}{MESSAGES_PATH}")
+    match stripped.split_once('?') {
+        Some((before, query)) if url_path(stripped).ends_with('/') => {
+            format!("{}?{query}", before.trim_end_matches('/'))
+        }
+        _ => stripped.to_string(),
+    }
+}
+
+/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as
+/// it is, query string included; anything else gets the route appended to the whole string.
+fn ensure_api_route(api_base: &str, route: &str) -> String {
+    let base = strip_trailing_slashes(api_base);
+    let route = route.trim_start_matches('/');
+    if url_path(&base).ends_with(&format!("/{route}")) {
+        return base;
+    }
+    format!("{}/{route}", strip_trailing_slashes(&base))
 }
 
 pub fn existing_betas(headers: &[(String, String)]) -> BetaSet {
@@ -1922,4 +1953,24 @@ mod tests {
             expected
         );
     }
+
+    #[rstest]
+    #[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")]
+    #[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")]
+    #[case::route_present(
+        "https://api.example.com/v1/messages/",
+        "https://api.example.com/v1/messages"
+    )]
+    #[case::route_and_query(
+        "https://api.example.com/v1/messages?beta=true",
+        "https://api.example.com/v1/messages?beta=true"
+    )]
+    #[case::query_without_route(
+        "https://api.example.com/?x=1",
+        "https://api.example.com?x=1/v1/messages"
+    )]
+    #[case::scheme_only("https://", "https:///v1/messages")]
+    fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) {
+        assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url);
+    }
 }

litellm-rust/crates/llms/src/anthropic/messages/thinking.rs

@@ -2,8 +2,8 @@ use litellm_llms_types::{
     formats::{
         chat_completions::ReasoningEffort,
         messages::{
-            EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig,
-            ThinkingDisplay,
+            AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig,
+            ThinkingConfig, ThinkingDisplay,
         },
     },
     recognized::Recognized,
@@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti
 
 fn effort_for_budget(
     budgets: &ThinkingBudgets,
-    budget_tokens: u64,
+    budget_tokens: i64,
     capabilities: &MessagesModelCapabilities,
 ) -> EffortLevel {
-    if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh {
+    let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier);
+    if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh {
         return EffortLevel::Xhigh;
     }
-    if budget_tokens >= budgets.high {
+    if reaches(budgets.high) {
         return EffortLevel::High;
     }
-    if budget_tokens >= budgets.medium {
+    if reaches(budgets.medium) {
         return EffortLevel::Medium;
     }
     EffortLevel::Low
@@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) -
     }
 }
 
+/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string
+/// as a budget, and nothing else.
+fn numeric_budget_tokens(budget: &Recognized<u64>) -> Option<i64> {
+    let value = match budget {
+        Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)),
+        Recognized::Unrecognized(value) => value,
+    };
+    match value {
+        Value::Number(number) => number
+            .as_i64()
+            .or_else(|| number.as_f64().map(|float| float.trunc() as i64)),
+        Value::String(text) => python_int(text),
+        Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None,
+    }
+}
+
+/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits.
+fn python_int(text: &str) -> Option<i64> {
+    let trimmed = text.trim();
+    let digits = trimmed.trim_start_matches(['+', '-']);
+    let well_formed = trimmed.len() - digits.len() <= 1
+        && !digits.is_empty()
+        && !digits.starts_with('_')
+        && !digits.ends_with('_')
+        && !digits.contains("__")
+        && digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_');
+    if !well_formed {
+        return None;
+    }
+    let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect();
+    cleaned.parse::<i64>().ok().or_else(|| {
+        Some(if cleaned.starts_with('-') {
+            i64::MIN
+        } else {
+            i64::MAX
+        })
+    })
+}
+
+fn python_type_name(value: &Value) -> &'static str {
+    match value {
+        Value::Null => "NoneType",
+        Value::Bool(_) => "bool",
+        Value::Number(number) if number.is_f64() => "float",
+        Value::Number(_) => "int",
+        Value::String(_) => "str",
+        Value::Array(_) => "list",
+        Value::Object(_) => "dict",
+    }
+}
+
+fn kept_display(
+    display: Option<&Recognized<ThinkingDisplay>>,
+    context: &ThinkingContext,
+) -> Option<Recognized<ThinkingDisplay>> {
+    match display? {
+        Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => {
+            display.cloned()
+        }
+        Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => {
+            display.cloned()
+        }
+        Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None,
+    }
+}
+
+/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive
+/// keys always, and any other key unless `drop_params` is set.
+fn carried_thinking_keys(
+    extra: &serde_json::Map<String, Value>,
+    context: &ThinkingContext,
+) -> serde_json::Map<String, Value> {
+    extra
+        .iter()
+        .filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params)
+        .map(|(key, value)| (key.clone(), value.clone()))
+        .collect()
+}
+
+/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`.
 fn translate_legacy_thinking_for_adaptive_model(
     request: MessagesRequest,
     context: &ThinkingContext,
-) -> MessagesRequest {
+) -> Result<MessagesRequest, Error> {
     let capabilities = &context.capabilities;
     if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking {
-        return request;
+        return Ok(request);
     }
     let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else {
-        return request;
+        return Ok(request);
     };
-    let budget = enabled
-        .budget_tokens
-        .as_ref()
-        .and_then(Recognized::known)
-        .copied()
-        .unwrap_or(0);
-    let level = effort_for_budget(&context.budgets, budget, capabilities);
-    MessagesRequest {
+    let budget = match &enabled.budget_tokens {
+        None => Some(0),
+        Some(budget) => numeric_budget_tokens(budget),
+    };
+    if budget.is_none() && !context.drop_params {
+        let actual = match &enabled.budget_tokens {
+            Some(Recognized::Unrecognized(value)) => python_type_name(value),
+            Some(Recognized::Known(_)) | None => "int",
+        };
+        return Err(Error::InvalidRequest(
+            crate::ErrorDetail::NonNumericThinkingBudget { actual },
+        ));
+    }
+    let thinking = ThinkingConfig::Adaptive(AdaptiveThinking {
+        display: kept_display(enabled.display.as_ref(), context),
+        extra: carried_thinking_keys(&enabled.extra, context),
+    });
+    let output_config = match budget {
+        Some(budget) => with_default_effort(
+            request.params.output_config,
+            effort_for_budget(&context.budgets, budget, capabilities),
+        ),
+        None => request.params.output_config,
+    };
+    Ok(MessagesRequest {
         params: MessagesOptionalParams {
-            thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))),
-            output_config: with_default_effort(request.params.output_config, level),
+            thinking: Some(Recognized::Known(thinking)),
+            output_config,
             ..request.params
         },
         ..request
-    }
+    })
 }
 
 fn translate_adaptive_effort_for_non_adaptive_model(
@@ -337,7 +435,7 @@ pub fn translate_thinking(
 ) -> Result<MessagesRequest, Error> {
     let request = translate_reasoning_effort(request, context)?;
     let request = drop_disabled_thinking(request, context);
-    let request = translate_legacy_thinking_for_adaptive_model(request, context);
+    let request = translate_legacy_thinking_for_adaptive_model(request, context)?;
     let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?;
     Ok(drop_incompatible_temperature_for_thinking(request, context))
 }
@@ -363,6 +461,7 @@ mod tests {
         ThinkingContext {
             capabilities,
             budgets: ThinkingBudgets::default(),
+            ..ThinkingContext::default()
         }
     }
 
@@ -1239,10 +1338,102 @@ mod tests {
         let context = ThinkingContext {
             capabilities,
             budgets: overridden_budgets(overrides),
+            ..ThinkingContext::default()
         };
         assert_eq!(
             translate_thinking(request(input), &context),
             Ok(request(expected))
         );
     }
+
+    use serde_json::json;
+
+    fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext {
+        ThinkingContext {
+            drop_params,
+            keeps_display_updates,
+            ..context(opus_4_7())
+        }
+    }
+
+    #[rstest]
+    #[case::float_budget(json!(8192.0), "xhigh")]
+    #[case::float_below_a_tier(json!(1023.9), "low")]
+    #[case::string_budget(json!(" 16_000 "), "xhigh")]
+    #[case::negative_budget(json!(-5), "low")]
+    fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) {
+        let translated = translate_thinking(
+            request(
+                json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}),
+            ),
+            &legacy_context(false, false),
+        );
+        assert_eq!(
+            translated,
+            Ok(request(json!({
+                "max_tokens": 64000,
+                "thinking": {"type": "adaptive"},
+                "output_config": {"effort": effort},
+            })))
+        );
+    }
+
+    #[rstest]
+    #[case::text(json!("lots"), "str")]
+    #[case::list(json!([1]), "list")]
+    #[case::null(Value::Null, "NoneType")]
+    #[case::boolean(json!(true), "bool")]
+    fn a_non_numeric_legacy_budget_is_rejected_unless_dropped(
+        #[case] budget: Value,
+        #[case] actual: &'static str,
+    ) {
+        let fields =
+            json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}});
+        assert_eq!(
+            translate_thinking(request(fields.clone()), &legacy_context(false, false)),
+            Err(Error::InvalidRequest(
+                crate::ErrorDetail::NonNumericThinkingBudget { actual }
+            ))
+        );
+        assert_eq!(
+            translate_thinking(request(fields), &legacy_context(true, false)),
+            Ok(request(
+                json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}})
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::summarized("summarized", false, Some("summarized"))]
+    #[case::updates_kept("updates", true, Some("updates"))]
+    #[case::updates_dropped("updates", false, None)]
+    #[case::unknown("verbose", true, None)]
+    fn legacy_display_survives_only_where_python_keeps_it(
+        #[case] display: &str,
+        #[case] keeps_display_updates: bool,
+        #[case] kept: Option<&str>,
+    ) {
+        let translated = translate_thinking(
+            request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})),
+            &legacy_context(false, keeps_display_updates),
+        )
+        .unwrap();
+        let thinking = serde_json::to_value(translated.params.thinking).unwrap();
+        assert_eq!(thinking.get("display").and_then(Value::as_str), kept);
+    }
+
+    #[rstest]
+    #[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))]
+    #[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))]
+    fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) {
+        let translated = translate_thinking(
+            request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})),
+            &legacy_context(drop_params, false),
+        )
+        .unwrap();
+        assert_eq!(
+            serde_json::to_value(translated.params.thinking).unwrap(),
+            expected
+        );
+    }
 }

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         request: MessagesRequest,
         context: &MessagesTransformContext,
     ) -> Result<MessagesRequest, Error> {
-        transform_messages_request(request, context)
+        transform_messages_request(request, &context.keeping_display_updates())
     }
 
     fn secret_names(&self) -> &'static [&'static str] {
@@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
             .then_some(AnthropicBeta::PerTurnControl20260701),
         uses_thinking_display_updates(params)
             .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
+        uses_thinking_block_binding(params)
+            .then_some(AnthropicBeta::ThinkingBindingControls20260801),
         messages_carry_tool_changes(&request.messages)
             .then_some(AnthropicBeta::MidConversationToolChanges20260701),
         has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
@@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
     display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
 }
 
+/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`.
+fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool {
+    match params.thinking.as_ref() {
+        Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => {
+            thinking.extra.contains_key("block_binding")
+        }
+        Some(Recognized::Unrecognized(Value::Object(thinking))) => {
+            thinking.contains_key("block_binding")
+        }
+        Some(Recognized::Unrecognized(_)) | None => false,
+    }
+}
+
 fn is_tool_reference_change(block: &ContentBlock) -> bool {
     matches!(
         &block.block_type,
@@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error
     })
 }
 
-fn drop_unsupported_params(
+/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`.
+pub fn drop_unsupported_params(
     request: MessagesRequest,
     context: &MessagesTransformContext,
 ) -> Result<MessagesRequest, Error> {
@@ -838,6 +860,8 @@ mod tests {
                             low: 2000,
                             ..ThinkingBudgets::default()
                         },
+                        drop_params: true,
+                        keeps_display_updates: false,
                     },
                     drop_params: true,
                 }

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
                     .capabilities
                     .supports_mid_conversation_system,
             )),
-            context,
+            &context.keeping_display_updates(),
         )
     }
 

litellm-rust/crates/llms/src/base_llm/messages/context.rs

@@ -110,6 +110,11 @@ impl ThinkingBudgets {
 pub struct ThinkingContext {
     pub capabilities: MessagesModelCapabilities,
     pub budgets: ThinkingBudgets,
+    /// `drop_params` as Python's thinking translation reads it.
+    pub drop_params: bool,
+    /// The provider accepts `display: "updates"` on thinking translated from the legacy shape
+    /// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`).
+    pub keeps_display_updates: bool,
 }
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub struct MessagesTransformContext {
@@ -130,8 +135,22 @@ impl MessagesTransformContext {
             thinking: ThinkingContext {
                 capabilities,
                 budgets: ThinkingBudgets::from_lookup(env),
+                drop_params,
+                keeps_display_updates: false,
             },
             drop_params,
         }
     }
+
+    /// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`.
+    #[must_use]
+    pub fn keeping_display_updates(&self) -> Self {
+        Self {
+            thinking: ThinkingContext {
+                keeps_display_updates: true,
+                ..self.thinking
+            },
+            ..*self
+        }
+    }
 }

litellm-rust/crates/llms/src/error.rs

@@ -92,6 +92,24 @@ impl Error {
         }
     }
 
+    pub fn upstream_headers(&self) -> &[(String, String)] {
+        match self {
+            Self::Http(error) => error.upstream_headers(),
+            _ => &[],
+        }
+    }
+
+    pub fn is_body_read_timeout(&self) -> bool {
+        matches!(self, Self::Http(error) if error.is_body_read_timeout())
+    }
+
+    pub fn is_timeout(&self) -> Option<bool> {
+        match self {
+            Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+            _ => None,
+        }
+    }
+
     pub fn upstream_response(&self) -> Option<(u16, &str)> {
         match self {
             Self::Http(error) => error.upstream_response(),
@@ -158,6 +176,10 @@ pub enum ErrorDetail {
         value: String,
         hint: String,
     },
+    #[error(
+        "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy"
+    )]
+    NonNumericThinkingBudget { actual: &'static str },
     #[error(
         "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params"
     )]

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         request: MessagesRequest,
         context: &MessagesTransformContext,
     ) -> Result<MessagesRequest, Error> {
-        let request = transform_messages_request(request, context)?;
+        let request = transform_messages_request(request, &context.keeping_display_updates())?;
         let request = strip_cache_control_scope(normalize_system_role_messages(
             request,
             context

litellm-rust/crates/model-catalog/AGENTS.md

@@ -1,3 +1,41 @@
+## Concepts
+
+Every price in a row lowers to one `Price { meter, channel, when }` and its `UnitPrice`. These keys come from the `FlatKey::parse` tests
+
+- `input_cost_per_token`
+  - meter `Input(Token(Text))`, channel `Online`, tier `Standard`, band `Base`
+- `output_cost_per_token_above_272k_tokens_priority`
+  - meter `Output(Token(Text))`, channel `Online`
+  - `_priority` sets tier `Priority`
+  - `_above_272k_tokens` sets band `OverPromptTokens(272000)`
+- `cache_creation_input_token_cost_above_1hr_above_200k_tokens`
+  - meter `CacheWrite { Text, OneHour }`, where `_above_1hr` is the cache TTL, not a band
+  - `_above_200k_tokens` sets band `OverPromptTokens(200000)`
+  - channel `Online`, tier `Standard`
+- `ocr_cost_per_page_batches`
+  - meter `Feature(OcrPage)`, tier `Standard`, band `Base`
+  - `_batches` sets channel `Batch`
+
+- Meter is what gets counted, such as text input tokens, reasoning tokens, an image of a given size or a web search
+  - A variation only one meter has stays inside it (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`)
+- `UnitPrice` is USD for one unit of the meter, such as one token, image, page or second
+- Band is a range of prompt sizes, starting where `_above_<N>_tokens` or `_above_<N>k_tokens` says
+  - `Prices::band` picks one band from prompt tokens alone, the highest edge passed wins, and `Band::Base` is below every edge
+  - The whole call moves, so every token meter, output included, bills all its tokens at that band's rate
+  - `Billing::band_edge` says whether a prompt exactly at the edge is in the band (exclusive by default, inclusive for xAI)
+  - A band only another tier or the batch channel prices never moves the call
+- `When` is the request's condition, its service tier and band
+  - A missing price falls back from `(tier, band)` to `(standard, band)`, `(tier, base)`, then `(standard, base)`
+- `Channel` is the route whose price list the call reads, and each falls back differently
+  - `Online` falls back across `When` as above
+  - `Batch` (`_batches`) uses bands but no service tier, and a missing price is half the online rate
+  - `Decisions` (`_decisions`, OpenAI `/v1/decisions`) overrides single online fields, and a missing price is the full online rate
+- Off-peak and `Adjustments` are never prices
+  - Off-peak swaps in other token rates while a UTC window is open
+  - `Adjustments` multiplies a priced call for region, residency, geo or `speed: fast`
+- `Billing` on `Prices` holds row attributes that change how its prices apply (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope)
+  - `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
+
 ## Layout
 
 `wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
@@ -6,8 +44,6 @@
 
 `catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
 
-`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
-
 `store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
 
 `layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
@@ -16,7 +52,7 @@ Catalog errors use one opaque `Error` with typed causes and JSON Pointer accesso
 
 ## Prices
 
-The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
+The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names
 
 A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`
 

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -186,6 +186,9 @@ pub struct ModelInfoFields {
     pub supports_image_size: Option<bool>,
     pub supported_audio_formats: Option<Value>,
     pub token_count_framing: Option<i64>,
+    pub token_count_tool_framing: Option<i64>,
+    pub token_count_responses_framing: Option<i64>,
+    pub token_count_responses_tool_framing: Option<i64>,
     pub token_count_model: Option<Value>,
     pub token_count_tokenizer: Option<Value>,
     pub vertex_ai_audio_api: Option<Value>,

litellm-rust/crates/model-catalog/src/vocabulary.rs

@@ -93,6 +93,12 @@ pub enum Endpoint {
     Realtime,
     #[serde(rename = "/v1/realtime/transcription_sessions")]
     RealtimeTranscriptionSessions,
+    #[serde(rename = "/v1/realtime/translations")]
+    RealtimeTranslations,
+    #[serde(rename = "/v1/realtime/translations/calls")]
+    RealtimeTranslationCalls,
+    #[serde(rename = "/v1/realtime/translations/client_secrets")]
+    RealtimeTranslationClientSecrets,
     #[serde(rename = "/v1/videos")]
     Videos,
     #[serde(rename = "/v1/ocr")]

litellm-rust/crates/model-catalog/src/wire/attributes.rs

@@ -11,7 +11,7 @@ use crate::error::{DateCause, Error, Repr};
 use crate::vocabulary::{
     AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
 };
-use crate::wire::lax::Count;
+use crate::wire::lax::{Count, NumericCount};
 
 macro_rules! attributes {
     ($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
@@ -93,6 +93,8 @@ attributes! {
     search_free_requests: SearchFreeRequests,
     /// URL of the provider pricing/model page this entry was taken from.
     source: String,
+    /// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists.
+    successor_model: String,
     /// Audio container formats the model can return.
     supported_audio_formats: Vec<AudioFormat>,
     /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
@@ -104,9 +106,14 @@ attributes! {
     /// Cloud regions the model is available in ('global' or region ids).
     supported_regions: Vec<String>,
     thinking_always_on: bool,
-    token_count_framing: u64 as Count,
+    /// Whether the model thinks when the request carries no thinking param.
+    thinking_on_by_default: bool,
+    token_count_framing: u64 as NumericCount,
     token_count_model: TokenCountModel,
+    token_count_responses_framing: u64 as NumericCount,
+    token_count_responses_tool_framing: u64 as NumericCount,
     token_count_tokenizer: TokenCountTokenizer,
+    token_count_tool_framing: u64 as NumericCount,
     /// Provider default tokens-per-minute limit.
     tpm: u64 as Count,
     use_openai_responses_path: bool,
@@ -298,10 +305,26 @@ mod tests {
     #[case::known_mode("mode", json!("chat"), true)]
     #[case::unknown_mode("mode", json!("chatt"), false)]
     #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
+    #[case::successor_model("successor_model", json!("gpt-6-luna"), true)]
+    #[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)]
+    #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
+    #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
     #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
     #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
     #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
     #[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
+    #[case::framing("token_count_framing", json!(12), true)]
+    #[case::framing_written_as_text("token_count_framing", json!("12"), false)]
+    #[case::framing_written_as_whole_float("token_count_framing", json!(12.0), true)]
+    #[case::fractional_tool_framing("token_count_tool_framing", json!(1.5), false)]
+    #[case::tool_framing("token_count_tool_framing", json!(214), true)]
+    #[case::tool_framing_written_as_text("token_count_tool_framing", json!("214"), false)]
+    #[case::tool_framing_as_a_flag("token_count_tool_framing", json!(true), false)]
+    #[case::negative_tool_framing("token_count_tool_framing", json!(-1), false)]
+    #[case::responses_framing("token_count_responses_framing", json!(38), true)]
+    #[case::responses_framing_as_a_flag("token_count_responses_framing", json!(false), false)]
+    #[case::responses_tool_framing("token_count_responses_tool_framing", json!(267), true)]
+    #[case::responses_tool_framing_written_as_text("token_count_responses_tool_framing", json!("267"), false)]
     fn each_attribute_reads_its_own_type(
         #[case] key: &str,
         #[case] value: Value,

litellm-rust/crates/model-catalog/src/wire/lax.rs

@@ -8,6 +8,8 @@ pub(crate) struct LaxI64;
 
 pub(crate) struct Count;
 
+pub(crate) struct NumericCount;
+
 impl<'de> DeserializeAs<'de, i64> for LaxI64 {
     fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
         deserializer.deserialize_any(Self)
@@ -52,6 +54,34 @@ impl<'de> DeserializeAs<'de, u64> for Count {
     }
 }
 
+impl<'de> DeserializeAs<'de, u64> for NumericCount {
+    fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
+        deserializer.deserialize_any(Self)
+    }
+}
+
+impl Visitor<'_> for NumericCount {
+    type Value = u64;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("a whole number of zero or more")
+    }
+
+    fn visit_u64<E: de::Error>(self, value: u64) -> Result<u64, E> {
+        Ok(value)
+    }
+
+    fn visit_i64<E: de::Error>(self, value: i64) -> Result<u64, E> {
+        u64::try_from(value).map_err(|_| E::invalid_value(de::Unexpected::Signed(value), &self))
+    }
+
+    fn visit_f64<E: de::Error>(self, value: f64) -> Result<u64, E> {
+        integral_float(value)
+            .and_then(|whole| u64::try_from(whole).ok())
+            .ok_or_else(|| E::invalid_value(de::Unexpected::Float(value), &self))
+    }
+}
+
 fn integer_string(value: &str) -> Option<i64> {
     let integer = match value.split_once('.') {
         Some((integer, fraction)) => {
@@ -99,6 +129,12 @@ mod tests {
             .map(DeserializeAsWrap::into_inner)
     }
 
+    fn numeric_count(value: Value) -> Option<u64> {
+        DeserializeAsWrap::<u64, NumericCount>::deserialize(value)
+            .ok()
+            .map(DeserializeAsWrap::into_inner)
+    }
+
     use serde::Deserialize;
 
     #[rstest]
@@ -125,4 +161,23 @@ mod tests {
     ) {
         assert_eq!(count(written), expected);
     }
+
+    #[rstest]
+    #[case::integer(json!(214), Some(214))]
+    #[case::zero(json!(0), Some(0))]
+    #[case::whole_float(json!(214.0), Some(214))]
+    #[case::negative_zero(json!(-0.0), Some(0))]
+    #[case::above_i64(json!(u64::MAX), Some(u64::MAX))]
+    #[case::negative(json!(-1), None)]
+    #[case::negative_whole_float(json!(-1.0), None)]
+    #[case::fraction(json!(1.5), None)]
+    #[case::string(json!("214"), None)]
+    #[case::boolean(json!(true), None)]
+    #[case::list(json!([214]), None)]
+    fn numeric_counts_accept_only_whole_json_numbers(
+        #[case] written: Value,
+        #[case] expected: Option<u64>,
+    ) {
+        assert_eq!(numeric_count(written), expected);
+    }
 }

litellm-rust/crates/python-bridge/src/errors.rs

@@ -23,6 +23,17 @@ pyo3::create_exception!(
 );
 
 const MESSAGES_REQUEST_ERROR: &str = "messages_request_error";
+/// The rejection is a parameter the model does not support, which Python raises as
+/// `UnsupportedParamsError`.
+const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param";
+/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or
+/// `Body` for a streamed body read that timed out after the head arrived.
+const MESSAGES_TIMEOUT: &str = "messages_timeout";
+/// No credential reached the provider, which Python raises as `AuthenticationError`.
+const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential";
+/// Python refuses the request only once its logger knows the resolved model and provider (the
+/// provider config's request transform), so the failure logs them.
+const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection";
 
 pub(crate) fn upstream_error(
     py: Python<'_>,
@@ -78,17 +89,57 @@ pub(crate) fn route_failure(
         return Ok(raised);
     }
     if let Some((status, body)) = error.upstream_response() {
-        return public(upstream_error(py, status, body, &HeaderMap::new())?);
+        let headers = error
+            .upstream_headers()
+            .iter()
+            .filter_map(|(name, value)| {
+                Some((
+                    reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?,
+                    reqwest::header::HeaderValue::from_str(value).ok()?,
+                ))
+            })
+            .collect::<HeaderMap>();
+        return public(upstream_error(py, status, body, &headers)?);
     }
     let native = match (marker, error.rejected_request()) {
         (RequestMarker::Messages, Some(rejection)) => {
             let native = PyValueError::new_err(Report::new(rejection).to_string());
-            native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?;
+            let value = native.value(py);
+            value.setattr(MESSAGES_REQUEST_ERROR, true)?;
+            if matches!(
+                rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
+                Some(
+                    litellm_llms::ErrorDetail::UnsupportedParameter { .. }
+                        | litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }
+                )
+            ) {
+                value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?;
+            }
+            if matches!(
+                rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
+                Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. })
+            ) {
+                value.setattr(MESSAGES_LATE_REJECTION, true)?;
+            }
             native
         }
-        (RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => {
-            by_fault(error.is_request(), &error)
+        (RequestMarker::Messages, None) => {
+            let native = by_fault(error.is_request(), &error);
+            let value = native.value(py);
+            if let Some(connect) = error.timeout() {
+                let phase = match (connect, error.body_read_timeout()) {
+                    (true, _) => "Connection",
+                    (false, true) => "Body",
+                    (false, false) => "Read",
+                };
+                value.setattr(MESSAGES_TIMEOUT, phase)?;
+            }
+            if error.is_missing_credential() {
+                value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?;
+            }
+            native
         }
+        (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error),
     };
     public(native)
 }

litellm-rust/crates/python-bridge/src/routes/messages/host.rs

@@ -34,11 +34,9 @@ fn merge_headers(
     forwarded: Option<Map<String, Value>>,
     extra_headers: Option<Map<String, Value>>,
 ) -> Option<Map<String, Value>> {
-    let merged: Map<String, Value> = forwarded
-        .into_iter()
-        .flatten()
-        .chain(extra_headers.into_iter().flatten())
-        .collect();
+    let merged = litellm_inference_messages::merge_messages_header_sources(
+        forwarded.into_iter().chain(extra_headers),
+    );
     (!merged.is_empty()).then_some(merged)
 }
 
@@ -107,10 +105,19 @@ impl MessagesPythonHost {
             .map(|value| python_timeout_seconds(py, value.unbind()))
             .transpose()?
             .flatten();
-        let custom_llm_provider = string("custom_llm_provider")?;
-        let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
         let api_key = string("api_key")?;
         let api_base = string("api_base")?;
+        // Python's handler resolves a bare model name (`claude-sonnet-4-5`) through
+        // `get_llm_provider`'s model lists, which the native prefix lookup does not carry.
+        let custom_llm_provider = match string("custom_llm_provider")? {
+            Some(provider) => Some(provider),
+            None => py
+                .import(ROUTE_HOST_MODULE)?
+                .getattr("inferred_provider")?
+                .call1((&model, api_base.as_deref()))?
+                .extract()?,
+        };
+        let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
         let extra_headers = self.merged_headers(py, arguments)?;
         let provider_specific_header = self.provider_specific_header(py, arguments)?;
         let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;

litellm-rust/crates/python-bridge/src/routes/messages/mod.rs

@@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
     )
 }
 
+/// Python's sync Messages handler raises for every native provider before it sends anything
+/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies
+/// that until both sides serve sync calls.
+const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls";
+
 #[pyfunction]
-pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
-    run_messages(py, call, false)
+#[pyo3(signature = (call))]
+pub(crate) fn messages(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
+    drop(call);
+    Err(pyo3::exceptions::PyValueError::new_err(
+        SYNC_NOT_IMPLEMENTED,
+    ))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -8,7 +8,7 @@ _apply_process_defaults()
 import threading
 import os
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from typing import (
     Any,
     Callable,
@@ -1562,7 +1562,9 @@ from .realtime_api.main import (
     _arealtime,
     acreate_realtime_client_secret,
     acreate_realtime_transcription_session,
+    acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
     arealtime_calls,
+    arealtime_translation_calls as arealtime_translation_calls,
 )
 from .responses.main import _aresponses_websocket
 
@@ -1647,6 +1649,7 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
 # Lazy loading system for heavy modules to reduce initial import time and memory usage
 
 if TYPE_CHECKING:
+    from . import guardrails as guardrails
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -1753,6 +1756,9 @@ if TYPE_CHECKING:
     from .llms.typesafe.decisions.transformation import (
         TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
     )
+    from .llms.parallel_ai.decisions.transformation import (
+        ParallelAIDecisionsConfig as ParallelAIDecisionsConfig,
+    )
     from .llms.openrouter.decisions.transformation import (
         OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
     )
@@ -1807,6 +1813,9 @@ if TYPE_CHECKING:
     from .llms.vertex_ai.rerank.transformation import (
         VertexAIRerankConfig as VertexAIRerankConfig,
     )
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.fireworks_ai.rerank.transformation import (
         FireworksAIRerankConfig as FireworksAIRerankConfig,
     )
@@ -1852,9 +1861,6 @@ if TYPE_CHECKING:
         BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
-    from .llms.together_ai.chat.transformation import (
-        TogetherAIChatConfig as TogetherAIChatConfig,
-    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -2013,6 +2019,9 @@ if TYPE_CHECKING:
     from .llms.xai.responses.transformation import (
         XAIResponsesAPIConfig as XAIResponsesAPIConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.litellm_proxy.responses.transformation import (
         LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
     )
@@ -2040,9 +2049,6 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
-    from .llms.vertex_ai.interactions.transformation import (
-        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
-    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,
@@ -2373,6 +2379,7 @@ if TYPE_CHECKING:
     supports_web_search: Callable[..., bool]
     supports_tool_search: Callable[..., bool]
     supports_url_context: Callable[..., bool]
+    supports_continuation: Callable[..., bool]
     supports_response_schema: Callable[..., bool]
     supports_parallel_function_calling: Callable[..., bool]
     supports_vision: Callable[..., bool]
@@ -2499,10 +2506,10 @@ def __getattr__(name: str) -> Any:
         handler_func: Final = registry[name]
         return handler_func(name)
 
-    if name == "proxy":
+    if name in ("proxy", "guardrails"):
         import importlib
 
-        return importlib.import_module("litellm.proxy")
+        return importlib.import_module(f"litellm.{name}")
 
     if name == "harness" or name in _AGENT_EXPORTS:
         import importlib

litellm/_internal_context.py

@@ -21,6 +21,7 @@ _T = TypeVar("_T")
 # When True, suppresses async logging and billing for internal sub-calls
 # (e.g., emulated file-search steps that make nested LLM calls).
 is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
+internal_parent_request_id: Final[ContextVar[str | None]] = ContextVar("internal_parent_request_id", default=None)
 
 # One request prices its totals, its per-token-type lines and the rates it reports on
 # separate code paths. Each reads the clock for off-peak pricing, so without a pinned
@@ -40,13 +41,27 @@ _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=Fa
 
 
 @contextmanager
-def internal_sub_call() -> Generator[None]:
+def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
     """A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
     token: Final = is_internal_call.set(True)
+    parent_token: Final = internal_parent_request_id.set(parent_request_id or internal_parent_request_id.get())
     try:
         yield
     finally:
         is_internal_call.reset(token)
+        internal_parent_request_id.reset(parent_token)
+
+
+@contextmanager
+def billed_separately() -> Generator[None]:
+    """A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
+    token: Final = is_internal_call.set(False)
+    parent_token: Final = internal_parent_request_id.set(None)
+    try:
+        yield
+    finally:
+        is_internal_call.reset(token)
+        internal_parent_request_id.reset(parent_token)
 
 
 @contextmanager

litellm/_lazy_imports_registry.py

@@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
     "supports_web_search",
     "supports_tool_search",
     "supports_url_context",
+    "supports_continuation",
     "supports_response_schema",
     "supports_parallel_function_calling",
     "supports_vision",
@@ -162,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
     "HostedVLLMRerankConfig",
     "PerplexityDecisionsConfig",
     "TypeSafeDecisionsConfig",
+    "ParallelAIDecisionsConfig",
     "NaceDecisionsConfig",
     "OpenRouterDecisionsConfig",
     "FireworksAIDecisionsConfig",
@@ -488,6 +490,7 @@ _UTILS_IMPORT_MAP: Final = {
     "supports_web_search": (".utils", "supports_web_search"),
     "supports_tool_search": (".utils", "supports_tool_search"),
     "supports_url_context": (".utils", "supports_url_context"),
+    "supports_continuation": (".utils", "supports_continuation"),
     "supports_response_schema": (".utils", "supports_response_schema"),
     "supports_parallel_function_calling": (
         ".utils",
@@ -738,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
     "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
     "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
+    "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
     "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
     "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
     "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),

litellm/accounting/protocols.py

@@ -3,7 +3,7 @@
 The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
 Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
 model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
-These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
+These inputs, including the effective cost-map rates, must survive request teardown.
 Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
 T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
 """
@@ -38,8 +38,8 @@ class CostCompiler(Protocol):
     async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
         """Reuse the committed charge or finalize one charge and its delivery intent.
 
-        Resolve captured request-scoped context by event identity and cost-map rates
-        at compilation. Persist the resolved inputs in the required snapshot. Replay
+        Resolve captured request-scoped context and rates by event identity.
+        Persist the resolved inputs in the required snapshot. Replay
         reuses the committed result and never consumes free allowance again.
         Missing pricing is Unpriced, never an inferred zero.
         """

litellm/accounting/types.py

@@ -276,8 +276,8 @@ class NamedCost(BaseModel):
 class PricingSnapshot(BaseModel):
     """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
 
-    Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
-    resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
+    Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
+    Retries reuse the committed charge and snapshot.
     Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
     Component sums and float tolerances belong to reconciliation, not this structural contract.
     """

litellm/anthropic_beta_headers_config.json

@@ -37,7 +37,47 @@
     "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "azure_ai": {
     "advisor-tool-2026-03-01": null,
@@ -45,23 +85,21 @@
     "bash_20241022": null,
     "bash_20250124": null,
     "code-execution-2025-08-25": "code-execution-2025-08-25",
-    "compact-2026-01-12": null,
-    "compact-2026-09-04": "compact-2026-09-04",
+    "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": null,
+    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": "mcp-client-2025-11-20",
     "mcp-client-2025-04-04": "mcp-client-2025-04-04",
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@@ -71,9 +109,54 @@
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": "web-fetch-2025-09-10",
-    "web-search-2025-03-05": "web-search-2025-03-05"
+    "web-search-2025-03-05": "web-search-2025-03-05",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "compact-2026-09-04": "compact-2026-09-04",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "bedrock_converse": {
     "advisor-tool-2026-03-01": null,
@@ -196,34 +279,92 @@
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": null,
     "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
-    "structured-outputs-2025-11-13": null,
+    "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": null,
+    "cache-diagnosis-2026-04-07": null,
+    "ce-plugins-2026-09-01": null,
+    "ce-user-management-2026-07-13": null,
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": null,
+    "compact-2026-09-04": null,
+    "computer-use-2024-10-22": null,
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": null,
+    "experimental-claude-tag-2026-10-21": null,
+    "extended-cache-ttl-2025-04-11": null,
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": null,
+    "mcp-client-2026-09-15": null,
+    "mcp-tunnels-2026-06-22": null,
+    "message-batches-2024-09-24": null,
+    "message-threads-2026-08-12": null,
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": null,
+    "output-300k-2026-03-24": null,
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": null,
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": null,
+    "spend-limit-reads-2026-09-26": null,
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": null,
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "user-profiles-2026-03-24": null,
+    "user-profiles-2026-08-18": null,
+    "user-profiles-2026-09-04": null,
+    "webhooks-2026-10-07": null,
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": null,
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "vertex_ai_header_only": {
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
     "compact-2026-09-04": "compact-2026-09-04",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04"
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,

litellm/anthropic_beta_headers_manager.py

@@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import cache
 from importlib.resources import files
 from typing import Final, Literal, TypeAlias
+from urllib.parse import urlparse
 
 import httpx
 from pydantic import TypeAdapter
@@ -48,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 _DECODED_JSON: Final = TypeAdapter(object)
 
 BetaPlacement: TypeAlias = Literal["header", "body"]
+UnlistedBetas: TypeAlias = Literal["drop", "forward"]
 _HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
 
 
@@ -280,12 +282,13 @@ def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
     placement: BetaPlacement = "header",
+    unlisted: UnlistedBetas = "drop",
 ) -> list[str]:
     """
     Filter and transform beta headers based on provider's mapping configuration.
 
     This function:
-    1. Only allows headers that are present in the provider's mapping keys
+    1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
     2. Filters out headers with null values (unsupported)
     3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
 
@@ -293,6 +296,7 @@ def filter_and_transform_beta_headers(
         beta_headers: List of Anthropic beta header values
         provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
         placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
+        unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
 
     Returns:
         List of filtered and transformed beta headers for the provider
@@ -307,6 +311,10 @@ def filter_and_transform_beta_headers(
     for header in beta_headers:
         header = header.strip()
 
+        if header not in provider_mapping and unlisted == "forward":
+            filtered_headers.add(header)
+            continue
+
         # Check if header is in the mapping
         if header not in provider_mapping:
             verbose_logger.debug(
@@ -388,9 +396,38 @@ def get_provider_beta_header(
     return provider_mapping[anthropic_beta_header]
 
 
+FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
+    "services.ai.azure.com",
+    "openai.azure.com",
+    "cognitiveservices.azure.com",
+)
+
+
+def is_foundry_api_base(api_base: str | None) -> bool:
+    """Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft."""
+    if not api_base:
+        return False
+    host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or ""
+    return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES)
+
+
+def resolve_beta_provider(provider: str, api_base: str | None = None) -> str:
+    """
+    Pick the beta allowlist row for one deployment, not one provider string.
+
+    An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and
+    gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com.
+    """
+    if provider == "anthropic" and is_foundry_api_base(api_base):
+        return "azure_ai"
+    return provider
+
+
 def update_headers_with_filtered_beta(
     headers: dict,
     provider: str,
+    api_base: str | None = None,
+    unlisted: UnlistedBetas = "drop",
 ) -> dict:
     """
     Update headers dict by filtering and transforming anthropic-beta header values.
@@ -399,6 +436,8 @@ def update_headers_with_filtered_beta(
     Args:
         headers: Request headers dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
+        unlisted: Whether a beta missing from the row is dropped or forwarded as sent
 
     Returns:
         Updated headers dict
@@ -413,7 +452,8 @@ def update_headers_with_filtered_beta(
     # Filter and transform based on provider
     filtered_beta_values: Final = filter_and_transform_beta_headers(
         beta_headers=beta_values,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
+        unlisted=unlisted,
     )
 
     # Update or remove the header
@@ -430,6 +470,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
+    api_base: str | None = None,
 ) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
@@ -439,11 +480,12 @@ def update_request_with_filtered_beta(
         headers: Request headers dict (will be modified in place)
         request_data: Request body dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
 
     Returns:
         Tuple of (updated headers, updated request_data)
     """
-    headers = update_headers_with_filtered_beta(headers=headers, provider=provider)
+    headers = update_headers_with_filtered_beta(headers=headers, provider=provider, api_base=api_base)
 
     existing_body_betas: Final = request_data.get("anthropic_beta")
     if not existing_body_betas:
@@ -451,7 +493,7 @@ def update_request_with_filtered_beta(
 
     filtered_body_betas: Final = filter_and_transform_beta_headers(
         beta_headers=existing_body_betas,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
         placement="body",
     )
 

litellm/anthropic_interface/exceptions/exception_mapping_utils.py

@@ -16,7 +16,7 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE
 _PARSED_ERROR_BODY: Final = TypeAdapter(object)
 
 # HTTP status code -> Anthropic error type
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
     400: "invalid_request_error",
     401: "authentication_error",
@@ -46,6 +46,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Create an Anthropic-formatted error response dict.
@@ -57,12 +58,10 @@ class AnthropicExceptionMapping:
             "request_id": "req_..."
         }
         """
-        error_type: Final = AnthropicExceptionMapping.get_error_type(status_code)
-
         response: Final[AnthropicErrorResponse] = {
             "type": "error",
             "error": {
-                "type": error_type,
+                "type": error_type or AnthropicExceptionMapping.get_error_type(status_code),
                 "message": message,
             },
         }
@@ -131,6 +130,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         raw_message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Transform an error message to Anthropic format.
@@ -144,6 +144,7 @@ class AnthropicExceptionMapping:
             status_code: HTTP status code
             raw_message: Raw error message (may be JSON string or plain text)
             request_id: Optional request ID to include
+            error_type: Anthropic error type to use instead of the one the status maps to
 
         Returns:
             AnthropicErrorResponse dict
@@ -169,6 +170,7 @@ class AnthropicExceptionMapping:
             status_code=status_code,
             message=message,
             request_id=request_id,
+            error_type=error_type,
         )
 
 

litellm/anthropic_interface/exceptions/exceptions.py

@@ -6,15 +6,18 @@ from typing import Literal
 from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 # Known Anthropic error types
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 AnthropicErrorType = Literal[
     "invalid_request_error",
     "authentication_error",
+    "billing_error",
     "permission_error",
     "not_found_error",
+    "conflict_error",
     "request_too_large",
     "rate_limit_error",
     "api_error",
+    "timeout_error",
     "overloaded_error",
 ]
 
@@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict):
     type: AnthropicErrorType
     message: str
     provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
+    litellm_error_type: NotRequired[ReadOnly[str]]
     litellm_call_id: NotRequired[ReadOnly[str]]
 
 

litellm/batches/batch_utils.py

@@ -3,9 +3,10 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from dataclasses import replace as dataclasses_replace
 from enum import Enum
+from itertools import groupby
 from typing import Any, Final, Literal, cast
 
-from pydantic import TypeAdapter
+from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -15,6 +16,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
 from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
 from litellm.llms.vertex_ai.batches.transformation import (
     is_native_vertex_batch_output_row,
+    native_vertex_batch_row_model,
     native_vertex_batch_row_stats,
 )
 from litellm.types.llms.openai import Batch
@@ -210,6 +212,7 @@ class _BatchOutputLineStats:
     cache_creation_tokens: int
     reasoning_tokens: int
     model: str | None
+    usage: Usage | None
 
 
 def _classify_output_line_stats(
@@ -268,7 +271,7 @@ def _compute_output_line_stats(
     raw_model: Final = response_body.get("model")
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
     completion_details: Final = usage.completion_tokens_details
-    line_prompt_cost, line_completion_cost = _output_line_cost(
+    line_prompt_cost, line_completion_cost = _output_line_cost_or_pending(
         response_body=response_body,
         usage=usage,
         custom_llm_provider=custom_llm_provider,
@@ -286,7 +289,15 @@ def _compute_output_line_stats(
         cache_read_tokens=prompt_details["cache_hit_tokens"],
         cache_creation_tokens=prompt_details["cache_creation_tokens"],
         reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
-        model=response_model,
+        model=response_model or model_name,
+        usage=(
+            usage.model_copy(update=ocr_usage.model_dump(exclude_none=True))
+            if (ocr_usage := _ocr_usage_info_from_response_body(response_body)) is not None
+            and ocr_usage.model_dump(exclude_none=True)
+            else usage
+            if response_body.get("usage") is not None or usage.total_tokens > 0
+            else None
+        ),
     )
 
 
@@ -298,6 +309,54 @@ def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> O
     return OCRUsageInfo.model_validate(raw_usage_info)
 
 
+def _output_line_cost_or_pending(
+    response_body: Mapping[str, object],
+    usage: Usage,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
+    model_name: str | None,
+    response_model: str | None,
+    model_info: ModelInfo | None,
+    vertex_location: str | None,
+) -> tuple[float, float]:
+    try:
+        return _output_line_cost(
+            response_body, usage, custom_llm_provider, model_name, response_model, model_info, vertex_location
+        )
+    except Exception:  # noqa: BLE001  # retain measured usage when existing batch pricing fails
+        verbose_logger.warning("Batch line pricing unavailable; retaining usage with the existing zero-cost fallback")
+        return 0.0, 0.0
+
+
+def _combined_reported_batch_usage(usages: tuple[Usage | None, ...]) -> Mapping[str, object] | None:
+    from litellm.cost_calculator import BaseTokenUsageProcessor
+
+    reported: Final = [usage for usage in usages if usage is not None]
+    if not reported:
+        return None
+    fields: Final = tuple(TypeAdapter(dict[str, JsonValue]).validate_python(usage.model_dump()) for usage in reported)
+    quantities: Final = {
+        key: sum(value for usage in fields if isinstance(value := usage.get(key), (int, float)))
+        for key in OCRUsageInfo.model_fields
+        if any(isinstance(usage.get(key), (int, float)) for usage in fields)
+    }
+    return {**BaseTokenUsageProcessor.combine_usage_objects(usage_objects=reported).model_dump(), **quantities}
+
+
+def _batch_usage_by_request(usages: tuple[tuple[str | None, Usage | None], ...]) -> tuple[Mapping[str, object], ...]:
+    rows: Final = tuple(
+        {"model": model, "usage": usage.model_dump() if usage is not None else None} for model, usage in usages
+    )
+    return tuple(sorted(rows, key=lambda row: json.dumps(row, sort_keys=True)))
+
+
+def _batch_usage_by_model(usages: tuple[tuple[str | None, Usage | None], ...]) -> Mapping[str, object]:
+    ordered: Final = sorted(usages, key=lambda item: item[0] or "")
+    return {
+        model: _combined_reported_batch_usage(tuple(item[1] for item in group))
+        for model, group in groupby(ordered, key=lambda item: item[0] or "")
+    }
+
+
 def _output_line_cost(
     response_body: Mapping[str, object],
     usage: Usage,
@@ -361,6 +420,9 @@ def _aggregate_batch_cost_usage_models(
         completion_tokens=sum(stats.completion_tokens for stats in line_stats),
         reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
         **cache_token_params,
+        batch_usage_by_model=_batch_usage_by_model(tuple((stats.model, stats.usage) for stats in line_stats)),
+        batch_usage_by_request=_batch_usage_by_request(tuple((stats.model, stats.usage) for stats in line_stats)),
+        batch_unknown_usage_requests=sum(stats.usage is None for stats in line_stats),
     )
     batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
     total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
@@ -401,17 +463,21 @@ def calculate_vertex_ai_batch_cost_and_usage(
     from litellm.cost_calculator import batch_cost_calculator
     from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
 
-    row_stats: Final = tuple(
-        native_vertex_batch_row_stats(
-            row,
-            model_name,
-            model_info=model_info,
-            calculate_usage=VertexGeminiConfig.calculate_usage,
-            cost_calculator=batch_cost_calculator,
-            vertex_location=vertex_location,
+    observations: Final = tuple(
+        (
+            native_vertex_batch_row_model(row, model_name),
+            native_vertex_batch_row_stats(
+                row,
+                model_name,
+                model_info=model_info,
+                calculate_usage=VertexGeminiConfig.calculate_usage,
+                cost_calculator=batch_cost_calculator,
+                vertex_location=vertex_location,
+            ),
         )
         for row in vertex_ai_batch_responses
     )
+    row_stats: Final = tuple(stats for _, stats in observations)
     priced: Final = tuple(stats for stats in row_stats if stats is not None)
     total_prompt_cost: Final = sum(stats.prompt_cost for stats in priced)
     total_completion_cost: Final = sum(stats.completion_cost for stats in priced)
@@ -435,6 +501,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
             total_tokens=total_tokens,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
+            batch_usage_by_model=_batch_usage_by_model(
+                tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
+            ),
+            batch_usage_by_request=_batch_usage_by_request(
+                tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
+            ),
+            batch_unknown_usage_requests=sum(stats is None for stats in row_stats),
         ),
         models=(
             [model_name]

litellm/caching/caching.py

@@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
 from litellm._logging import verbose_logger
 from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
 from litellm.integrations.otel.runtime import phase_span
-from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
+from litellm.litellm_core_utils.model_param_helper import (
+    ModelParamHelper,
+    embeds_thought_signature_in_tool_call_id,
+    peek_reasoning_summary_aliases,
+)
 from litellm.types.caching import *
 from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
 
@@ -41,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
 from .s3_cache import S3Cache
 
 RESPONSE_CACHE_TARGET: Final = "llm_response"
-_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
+_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
 _DECODED_LITERAL: Final = TypeAdapter(object)
 _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
     list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
 )
 
 
+def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
+    raw_extra_body: Final = kwargs.get("extra_body")
+    if not isinstance(raw_extra_body, dict):
+        return ""
+    extra_body: Final = cast(  # cast-ok: isinstance leaves the key and value types unknown
+        Mapping[str, object], raw_extra_body
+    )
+    return "".join(
+        f"extra_body.{param}: {extra_body[param]}"
+        for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
+        if extra_body.get(param)
+    )
+
+
 @contextmanager
 def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
     """The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
@@ -421,7 +439,15 @@ class Cache:
         reasoning_summary_material: Final = (
             "" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
         )
-        hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
+        tool_call_id_shape_material: Final = (
+            "" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
+        )
+        hashed_cache_key = Cache._get_hashed_cache_key(
+            cache_key
+            + _keyed_provider_params_in_extra_body(kwargs)
+            + reasoning_summary_material
+            + tool_call_id_shape_material
+        )
         hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
         verbose_logger.debug(
             "\nCreated cache key: %s (source material length: %d)",

litellm/caching/dual_cache.py

@@ -673,7 +673,7 @@ class DualCache(BaseCache):
         if self.redis_cache is not None:
             self.redis_cache.flush_cache()
 
-    def delete_cache(self, key):
+    def delete_cache(self, key: str) -> None:
         """
         Delete a key from the cache
         """

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -11,7 +11,7 @@ from typing import (
     cast,  # noqa: TID251  # optional_params reaches the bridge as an untyped dict
 )
 
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.types.llms.openai import ResponsesAPIResponse
 
@@ -20,15 +20,15 @@ if TYPE_CHECKING:
 
 
 class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
-    model: str
-    messages: list
-    optional_params: dict
-    litellm_params: dict
-    headers: dict
-    model_response: "ModelResponse"
-    logging_obj: "LiteLLMLoggingObj"
-    custom_llm_provider: str
-    encoding: object
+    model: ReadOnly[str]
+    messages: ReadOnly[list]
+    optional_params: ReadOnly[dict[str, object]]
+    litellm_params: ReadOnly[dict[str, object]]
+    headers: ReadOnly[dict]
+    model_response: ReadOnly["ModelResponse"]
+    logging_obj: ReadOnly["LiteLLMLoggingObj"]
+    custom_llm_provider: ReadOnly[str]
+    encoding: ReadOnly[object]
 
 
 def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler:
         custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
         if kwargs.get("stream") is True and "stream" not in optional_params:
             optional_params = {**optional_params, "stream": True}
+        logging_obj.record_bridged_chat_request(optional_params, litellm_params)
 
         request_data: Final = self.transformation_handler.transform_request(
             model=model,
@@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler:
         custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
         if kwargs.get("stream") is True and "stream" not in optional_params:
             optional_params = {**optional_params, "stream": True}
+        logging_obj.record_bridged_chat_request(optional_params, litellm_params)
 
         try:
             request_data: Final = self.transformation_handler.transform_request(

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import (
     ResponseCustomToolCallOutputParam,
     ResponseCustomToolCallParam,
 )
+from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
 from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
 from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
 from openai.types.responses.tool_param import FunctionToolParam
@@ -33,11 +34,17 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
     replay_items,
     requested_phase,
 )
-from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
+from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
+from litellm.litellm_core_utils.allowed_tools_tool_choice import (
+    parse_chat_allowed_tools_tool_choice,
+    responses_allowed_tools_tool_choice,
+)
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    prompt_cache_breakpoint_for_wire,
+    prompt_cache_breakpoint_with_readable_mode,
     responses_reasoning_items_from_thinking_blocks,
+    whole_prompt_cache_breakpoint_or_none,
     with_prompt_cache_breakpoint,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -97,6 +104,12 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
 )
 
 
+def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
+    if target_trims_breakpoints:
+        return prompt_cache_breakpoint_with_readable_mode(marker, drop_params)
+    return whole_prompt_cache_breakpoint_or_none(marker) if drop_params else marker
+
+
 def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
     if not isinstance(value, dict):
         return value
@@ -775,13 +788,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def _normalize_tool_choice_for_responses_api(
         self, tool_choice: _ToolChoiceT
-    ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
+    ) -> (
+        _ToolChoiceT
+        | ToolChoiceFunctionParam
+        | ToolChoiceCustomParam
+        | ToolChoiceAllowedParam
+        | Literal["auto", "none", "required"]
+    ):
         """Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
         if not isinstance(tool_choice, dict):
             return tool_choice
         choice_type: Final = tool_choice.get("type")
         if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
             return choice_type
+        if choice_type == "allowed_tools":
+            parsed: Final = parse_chat_allowed_tools_tool_choice(
+                cast(  # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
+                    object, tool_choice
+                )
+            )
+            flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None
+            if flat is not None:
+                return flat
         if choice_type not in ("function", "custom"):
             return tool_choice
         if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
@@ -844,6 +872,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
@@ -852,6 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
+            target_trims_breakpoints=target_trims_breakpoints,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
             replay_openai_history=replay_openai_history,
@@ -868,6 +898,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
         replay_openai_history: bool = False,
@@ -921,6 +952,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 content,
                                 role,
                                 drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                         }
                     )
@@ -940,6 +972,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         content,
                         "user",  # Use "user" role to get input_* types
                         drop_params=drop_params,
+                        target_trims_breakpoints=target_trims_breakpoints,
                     )
                 else:
                     # Fallback: convert unexpected types to input_text
@@ -968,7 +1001,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "type": "message",
                             "role": "assistant",
                             "content": self._convert_content_to_responses_format(
-                                content, "assistant", drop_params=drop_params
+                                content,
+                                "assistant",
+                                drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                             **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
@@ -1009,7 +1045,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(
-                            content, cast(str, role), drop_params=drop_params
+                            content,
+                            cast(str, role),
+                            drop_params=drop_params,
+                            target_trims_breakpoints=target_trims_breakpoints,
                         ),
                         **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
@@ -1127,14 +1166,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         optional_params = self._extract_extra_body_params(optional_params)
         provider: Final = _served_provider(litellm_params)
         base_model: Final = litellm_params.get("base_model")
-        supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or (
-            isinstance(base_model, str)
-            and bool(base_model)
-            and supports_openai_prompt_cache_breakpoint(base_model, provider)
+        model_info: Final = routed_model_info(litellm_params.get("model_info"))
+        named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,)
+        supports_prompt_cache_breakpoint: Final = any(
+            supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
         )
         converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
             messages,
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
+            target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
@@ -1195,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         api_model: Final = model
 
-        from litellm.types.utils import CallTypes
-
-        setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
-
         sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
 
         request_data: Final = {
@@ -1640,6 +1676,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         | None,
         role: str,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
     ) -> list[dict[str, object]]:
         """Convert chat completion content to responses API format"""
         from litellm.types.llms.openai import ChatCompletionImageObject
@@ -1666,7 +1703,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     if original_type == "text":
                         converted = with_prompt_cache_breakpoint(
                             self._convert_content_str_to_input_text(item.get("text", ""), role),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
@@ -1679,14 +1718,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                     cast(ChatCompletionImageObject, item), role
                                 ),
                             ),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
                     elif original_type == "input_audio":
                         converted = with_prompt_cache_breakpoint(
                             {"type": "input_audio", "input_audio": item.get("input_audio")},
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   input_audio -> %s", converted)
@@ -1702,7 +1745,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 _input_file_from_file_value(
                                     cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
                                 ),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1726,7 +1771,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             # Default to input_text for unknown types
                             converted = with_prompt_cache_breakpoint(
                                 self._convert_content_str_to_input_text(str(item.get("text", item)), role),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   unknown(%s) -> %s", original_type, converted)

litellm/constants.py

@@ -432,6 +432,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
 DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
+DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
 
 BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
@@ -450,6 +451,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
     {"response.completed", "response.incomplete", "response.failed"}
 )
 REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
+REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally"
 
 # SSL/TLS cipher configuration for faster handshakes
 # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
@@ -675,7 +677,7 @@ request_timeout: float = (
     DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env
 )
 request_timeout_explicitly_set: bool = _request_timeout_from_env is not None
-DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)  # 10 minutes
+DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)
 AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0
 AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000
 # Patterns that indicate a localhost/internal URL in A2A agent cards that should be
@@ -698,6 +700,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
 STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
 STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
+STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
+STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
+VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
     "DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400
@@ -967,7 +972,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
 
 OPENAI_TRANSCRIPTION_PARAMS: Final = [
     "language",
+    "languages",
+    "keywords",
     "response_format",
+    "stream",
     "timestamp_granularities",
 ]
 
@@ -1691,6 +1699,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
 VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
 ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
+RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
 BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"

litellm/cost_calculator.py

@@ -6,6 +6,7 @@ import time
 from collections.abc import Mapping, Sequence
 from datetime import datetime
 from functools import lru_cache
+from itertools import accumulate
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
@@ -22,9 +23,14 @@ from litellm.constants import (
     OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
     REALTIME_METERED_USAGE_KEY,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
+from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
-from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
+from litellm.litellm_core_utils.decisions_pricing import (
+    decisions_pricing_fields,
+    decisions_request_price,
+    is_decisions_call_type,
+)
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
@@ -319,12 +325,41 @@ def _get_additional_costs(
     return None
 
 
-def _transcription_usage_has_token_details(
+_TranscriptionRateKey: TypeAlias = Literal[
+    "input_cost_per_second",
+    "output_cost_per_second",
+    "input_cost_per_audio_token",
+    "output_cost_per_audio_token",
+    "input_cost_per_token",
+    "output_cost_per_token",
+]
+_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_audio_token",
+    "output_cost_per_audio_token",
+)
+_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
+    "input_cost_per_second",
+    "output_cost_per_second",
+)
+
+
+def _transcription_uses_token_pricing(
+    model: str,
+    custom_llm_provider: str | None,
     usage_block: Usage | None,
 ) -> bool:
     if usage_block is None:
         return False
 
+    model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
+    if (
+        _declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
+        and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
+    ):
+        return False
+
     prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
     completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
     prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
@@ -338,13 +373,38 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
-def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
-    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
+    if pricing_model is None or custom_llm_provider is None:
         return False
+    transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
+    transcription_metadata: Final = (
+        as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
+    )
+    return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
+
+
+def _transcription_duration_for_cost(
+    completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
+) -> float:
+    hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
+    measured_duration: Final = normalized_audio_duration_seconds(
+        hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
+    )
+    if measured_duration is not None and measured_duration > 0:
+        return measured_duration
+    provider_duration: Final = _get_transcription_usage_duration(completion_response)
+    if (
+        provider_duration is not None
+        and provider_duration > 0
+        and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
+    ):
+        return provider_duration
+    if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
+        return 0.0
     transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
         model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
     )
-    return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
+    return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
 
 
 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@@ -604,6 +664,9 @@ def cost_per_token(
         decisions_overrides: Final = decisions_pricing_fields(
             litellm.model_cost.get(decisions_model_info["key"], {})  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # model cost map is dynamically loaded
         )
+        request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides)
+        if request_price is not None:
+            return request_price, 0.0
         if decisions_overrides:
             return generic_cost_per_token(
                 model=model,
@@ -693,7 +756,7 @@ def cost_per_token(
             vertex_location=vertex_location,
         )
     elif call_type == "atranscription" or call_type == "transcription":
-        if _transcription_usage_has_token_details(usage_block):
+        if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
             return generic_cost_per_token(
                 model=model,
                 usage=usage_block,
@@ -1039,20 +1102,19 @@ def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_nam
     return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
 
 
-def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
-    rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
-    return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
+def declared_cost_map_pricing(key: str, model: str, custom_llm_provider: str | None) -> Mapping[str, object]:
+    entry: Final = _raw_cost_map_entry(key)
+    if entry is None:
+        return unmapped_model_generalization(model, custom_llm_provider) or MappingProxyType({})
+    fill_missing: Final = mapped_model_fill_missing(key, entry)
+    return entry if fill_missing is None else MappingProxyType({**fill_missing, **entry})
 
 
 def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
     key: Final = cached_cost_map_key(model, custom_llm_provider)
-    if key is None:
-        return False
-    entry: Final = _raw_cost_map_entry(key)
-    if entry is None:
-        return _rule_declares_rates(model, custom_llm_provider)
-    fill_missing: Final = mapped_model_fill_missing(key, entry)
-    return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
+    return key is not None and _cost_map_entry_prices_anything(
+        declared_cost_map_pricing(key, model, custom_llm_provider)
+    )
 
 
 def _served_or_priced_requested_model(
@@ -1275,6 +1337,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
     )
 
 
+def _get_transcription_usage_duration(completion_response: object) -> float | None:
+    response_fields: Final = as_str_mapping(completion_response)
+    usage_object: Final = (
+        response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
+    )
+    usage_fields: Final = as_str_mapping(usage_object)
+    usage_type: Final = (
+        usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
+    )
+    if usage_type != "duration":
+        return None
+    seconds: Final = (
+        usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
+    )
+    return normalized_audio_duration_seconds(seconds)
+
+
 def _extraction_usage(usage: ExtractionUsage) -> Usage:
     return Usage(
         prompt_tokens=usage.input_tokens,
@@ -1554,12 +1633,66 @@ def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float
     return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
 
 
+def _cost_map_priced_input_output(
+    model: str | None,
+    custom_llm_provider: str | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    if not model or usage is None:
+        return 0.0, 0.0
+    try:
+        return cost_per_token(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            usage_object=usage,
+            call_type=call_type or "completion",
+            service_tier=service_tier,
+            data_residency=data_residency,
+        )
+    except Exception as pricing_error:  # noqa: BLE001  # cost_per_token raises plain Exception for an unmapped model
+        verbose_logger.debug("No cost map price for reported-cost split of %s: %s", model, pricing_error)
+        return 0.0, 0.0
+
+
+def split_reported_cost_by_cost_map_ratio(
+    reported_total: float,
+    model: str | None,
+    custom_llm_provider: str | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    """Apportions a provider-reported total between input and output in the ratio litellm's own cost map would
+    price the same usage, so a per-input-token model books the whole total on input. Unmapped or zero-priced
+    usage keeps the whole total on output."""
+    priced_input, priced_output = _cost_map_priced_input_output(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        usage=usage,
+        call_type=call_type,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
+    priced_total: Final = priced_input + priced_output
+    if priced_total <= 0:
+        return 0.0, reported_total
+    input_cost: Final = reported_total * (priced_input / priced_total)
+    return input_cost, reported_total - input_cost
+
+
 def _adjusted_provider_billed_cost(
     reported_cost: float,
     custom_llm_provider: str | None,
+    pricing_provider: str | None,
     model: str | None,
     region_name: str | None,
     litellm_logging_obj: LitellmLoggingObject | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
     service_tier: str | None,
     data_residency: str | None,
 ) -> float:
@@ -1569,10 +1702,19 @@ def _adjusted_provider_billed_cost(
     adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
         base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
     )
+    input_cost, output_cost = split_reported_cost_by_cost_map_ratio(
+        reported_total=adjusted_cost,
+        model=model,
+        custom_llm_provider=pricing_provider,
+        usage=usage,
+        call_type=call_type,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
     _store_cost_breakdown_in_logging_obj(
         litellm_logging_obj=litellm_logging_obj,
-        prompt_tokens_cost_usd_dollar=0.0,
-        completion_tokens_cost_usd_dollar=adjusted_cost,
+        prompt_tokens_cost_usd_dollar=input_cost,
+        completion_tokens_cost_usd_dollar=output_cost,
         cost_for_built_in_tools_cost_usd_dollar=0.0,
         total_cost_usd_dollar=adjusted_cost,
         original_cost=reported_cost,
@@ -1937,20 +2079,8 @@ def completion_cost(
                         prompt,
                     )
                 elif call_type in _TRANSCRIPTION_CALL_TYPES:
-                    # Check _hidden_params first (duration stored there to
-                    # avoid polluting the response body), then fall back to
-                    # the response attribute (for verbose_json responses that
-                    # naturally include duration from the provider).
-                    _hidden = getattr(completion_response, "_hidden_params", {}) or {}
-                    audio_transcription_file_duration = _hidden.get(
-                        "audio_transcription_duration",
-                        getattr(completion_response, "duration", 0.0),
-                    ) or (
-                        completion_response.usage.seconds
-                        if isinstance(completion_response, TranscriptionResponse)
-                        and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
-                        and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
-                        else 0.0
+                    audio_transcription_file_duration = _transcription_duration_for_cost(
+                        completion_response, model, requested_model, custom_llm_provider
                     )
                 elif call_type in _RERANK_CALL_TYPES:
                     if completion_response is not None and isinstance(completion_response, RerankResponse):
@@ -2060,6 +2190,7 @@ def completion_cost(
                         litellm_logging_obj=litellm_logging_obj,
                         custom_pricing_model=selected_model if custom_pricing else None,
                         base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
+                        deployment_models=(base_model, requested_model),
                     )
                 elif call_type == _MCP_CALL_TYPE:
                     from litellm.proxy.mcp_server.cost_calculator import (
@@ -2080,9 +2211,16 @@ def completion_cost(
                     return _adjusted_provider_billed_cost(
                         reported_cost=reported_cost,
                         custom_llm_provider=custom_llm_provider,
+                        pricing_provider=_pricing_provider_for_cost_candidate(
+                            candidate_provider=candidate_pricing_provider,
+                            request_provider=request_pricing_provider,
+                            current_provider=custom_llm_provider,
+                        ),
                         model=model_for_discount,
                         region_name=region_name if isinstance(region_name, str) else None,
                         litellm_logging_obj=litellm_logging_obj,
+                        usage=cost_per_token_usage_object,
+                        call_type=call_type,
                         service_tier=service_tier,
                         data_residency=data_residency,
                     )
@@ -2281,18 +2419,20 @@ def completion_cost(
         raise e
 
 
-def get_response_cost_from_hidden_params(
-    hidden_params: dict | BaseModel,
-) -> float | None:
-    if isinstance(hidden_params, BaseModel):
-        _hidden_params_dict = hidden_params.model_dump()
-    else:
-        _hidden_params_dict = hidden_params
+_ADDITIONAL_HEADERS: Final = TypeAdapter(Mapping[str, object])
 
-    additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
-    if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
-        return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
-    return None
+
+def get_response_cost_from_hidden_params(
+    hidden_params: Mapping[str, object] | BaseModel,
+) -> float | None:
+    params: Final[Mapping[str, object]] = (
+        hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params
+    )
+    additional_headers: Final = params.get("additional_headers")
+    if not isinstance(additional_headers, Mapping):
+        return None
+    headers: Final = _ADDITIONAL_HEADERS.validate_python(additional_headers)
+    return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost"))
 
 
 def _billable_header_cost(value: object) -> float | None:
@@ -2387,10 +2527,28 @@ def response_cost_calculator(
                     provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
                     if provider_response_cost is not None and not custom_pricing:
                         if _breakdown_total(litellm_logging_obj) != provider_response_cost:
+                            header_priced_model: Final = select_model_name_for_cost_calc(
+                                model=model,
+                                completion_response=response_object,
+                                custom_llm_provider=custom_llm_provider,
+                                custom_pricing=custom_pricing,
+                                base_model=base_model,
+                                router_model_id=router_model_id,
+                                region_name=region_name,
+                            )
+                            header_input_cost, header_output_cost = split_reported_cost_by_cost_map_ratio(
+                                reported_total=provider_response_cost,
+                                model=header_priced_model or model,
+                                custom_llm_provider=pricing_provider or custom_llm_provider,
+                                usage=get_usage_object(response_object),
+                                call_type=call_type,
+                                service_tier=service_tier,
+                                data_residency=data_residency,
+                            )
                             _store_cost_breakdown_in_logging_obj(
                                 litellm_logging_obj=litellm_logging_obj,
-                                prompt_tokens_cost_usd_dollar=0.0,
-                                completion_tokens_cost_usd_dollar=provider_response_cost,
+                                prompt_tokens_cost_usd_dollar=header_input_cost,
+                                completion_tokens_cost_usd_dollar=header_output_cost,
                                 cost_for_built_in_tools_cost_usd_dollar=0.0,
                                 total_cost_usd_dollar=provider_response_cost,
                                 original_cost=provider_response_cost,
@@ -3313,13 +3471,16 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
     def create_logging_realtime_object(
         usage: Usage, results: OpenAIRealtimeStreamList
     ) -> LiteLLMRealtimeStreamLoggingObject:
+        translated_input_seconds, _ = _translation_session_seconds(results)
         return LiteLLMRealtimeStreamLoggingObject(
-            usage=usage,
+            usage=usage.model_copy(update={"seconds": translated_input_seconds})
+            if translated_input_seconds > 0
+            else usage,
             results=results,
         )
 
 
-_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
+_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
 
 
 class _ResponsesWsEventResponse(LiteLLMBaseModel):
@@ -3399,6 +3560,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
+_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
 
 
 class _LiveResponsePayload(BaseModel):
@@ -3621,6 +3783,7 @@ def handle_realtime_stream_cost_calculation(
     litellm_logging_obj: LitellmLoggingObject | None = None,
     custom_pricing_model: str | None = None,
     base_pricing_model: str | None = None,
+    deployment_models: Sequence[str | None] = (),
 ) -> float:
     """
     Handles the cost calculation for realtime stream responses.
@@ -3665,6 +3828,13 @@ def handle_realtime_stream_cost_calculation(
         if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
         else 0.0
     )
+    translation_cost: Final = handle_realtime_translation_cost_calculation(
+        results=results,
+        custom_llm_provider=custom_llm_provider,
+        litellm_model_name=litellm_model_name,
+        deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
+        session_model=received_model,
+    )
     live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
     live_session_started: Final = any(result["type"] == "session.started" for result in results)
     live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
@@ -3721,12 +3891,19 @@ def handle_realtime_stream_cost_calculation(
     input_cost: Final = input_cost_per_token + live_response_input_cost
     output_cost: Final = output_cost_per_token + live_response_output_cost
     total_cost: Final = (
-        input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
+        input_cost
+        + output_cost
+        + transcription_cost
+        + translation_cost
+        + live_voice_cost
+        + audio_duration_cost
+        + text_message_cost
     )
     additional_costs: Final = tuple(
         (cost_name, cost)
         for cost_name, cost in (
             ("transcription_cost", transcription_cost),
+            ("translation_cost", translation_cost),
             ("live_voice_cost", live_voice_cost),
             ("audio_duration_cost", audio_duration_cost),
             ("text_message_cost", text_message_cost),
@@ -3747,6 +3924,55 @@ def handle_realtime_stream_cost_calculation(
     return total_cost
 
 
+def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
+    usage_events: Final = tuple(
+        usage
+        for result in results
+        if (event := as_str_mapping(result)) is not None
+        and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
+        and (usage := as_str_mapping(event.get("usage"))) is not None
+    )
+    input_seconds: Final = sum(
+        seconds
+        for usage in usage_events
+        if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
+    )
+    output_seconds: Final = sum(
+        output_duration
+        for usage in usage_events
+        if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
+    )
+    return float(input_seconds), float(output_seconds)
+
+
+def _translation_per_second_rates(
+    models: Sequence[str | None], custom_llm_provider: str
+) -> tuple[float | None, float | None]:
+    model_infos: Final = tuple(
+        _lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
+    )
+    return (
+        _first_declared_transcription_rate(("input_cost_per_second",), model_infos),
+        _first_declared_transcription_rate(("output_cost_per_second",), model_infos),
+    )
+
+
+def handle_realtime_translation_cost_calculation(
+    results: OpenAIRealtimeStreamList,
+    custom_llm_provider: str,
+    litellm_model_name: str,
+    deployment_pricing_models: Sequence[str | None] = (),
+    session_model: str | None = None,
+) -> float:
+    input_seconds, output_seconds = _translation_session_seconds(results)
+    if input_seconds <= 0 and output_seconds <= 0:
+        return 0.0
+    input_rate, output_rate = _translation_per_second_rates(
+        (*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
+    )
+    return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
+
+
 def handle_realtime_transcription_cost_calculation(
     results: OpenAIRealtimeStreamList,
     custom_llm_provider: str,
@@ -3762,57 +3988,77 @@ def handle_realtime_transcription_cost_calculation(
       - {"type": "duration", "seconds": <float>}  → priced via input_cost_per_second
       - {"type": "tokens", "input_tokens": ...}    → priced via input/audio token cost
     """
-    completed_events: Final = [
-        cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
-    ]
-    if not completed_events:
+    configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
+    clips: Final = tuple(
+        (as_str_mapping(result.get("usage")) or {}, configured)
+        for result, configured in zip(results, configured_models, strict=True)
+        if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
+    )
+    if not clips:
         return 0.0
 
-    model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
-    model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
+    fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
+    clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
+    model_infos: Final = MappingProxyType(
+        {model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
+    )
     override_info: Final = (
         _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
     )
+    return sum(
+        (_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
+        0.0,
+    )
 
-    total_cost = 0.0
-    for event in completed_events:
-        usage = event.get("usage") or {}
-        total_cost += _transcription_usage_cost(usage, model_info, override_info)
-    return total_cost
+
+_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
+    {"transcription_session.created", "transcription_session.updated", "session.created", "session.updated"}
+)
+
+
+def _str_field(mapping: Mapping[str, object] | None, key: str) -> str | None:
+    value: Final = None if mapping is None else mapping.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _configured_transcription_model(session: Mapping[str, object]) -> str | None:
+    audio_input: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("input"))
+    nested: Final = _str_field(as_str_mapping((audio_input or {}).get("transcription")), "model")
+    return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
+
+
+def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
+    return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
+
+
+def _session_transcription_model(result: Mapping[str, object]) -> str | None:
+    session: Final = _session_of(result)
+    return None if session is None else _configured_transcription_model(session)
+
+
+def _newer_model(current: str | None, newer: str | None) -> str | None:
+    return newer or current
 
 
 def _get_transcription_model_name_from_results(
     results: OpenAIRealtimeStreamList,
 ) -> str | None:
-    """Resolve the ASR model from a transcription_session.* / session.* event."""
-    for result in results:
-        if result.get("type") in (
-            "transcription_session.created",
-            "transcription_session.updated",
-            "session.created",
-            "session.updated",
-        ):
-            session = cast(dict, result).get("session", {}) or {}
-            transcription = ((session.get("audio", {}) or {}).get("input", {}) or {}).get(
-                "transcription", {}
-            ) or session.get("input_audio_transcription", {})
-            model = (transcription or {}).get("model") or session.get("model")
-            if model:
-                return model
-    return None
+    return next(
+        (model for result in results if (model := _str_field(_session_of(result), "model"))),
+        None,
+    )
 
 
-_TranscriptionRateKey: TypeAlias = Literal[
-    "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
-]
-
-
-def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
+def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
     match key:
         case "input_cost_per_second":
             return info.get("input_cost_per_second")
+        case "output_cost_per_second":
+            return info.get("output_cost_per_second")
         case "input_cost_per_audio_token":
             return info.get("input_cost_per_audio_token")
+        case "output_cost_per_audio_token":
+            return info.get("output_cost_per_audio_token")
         case "input_cost_per_token":
             return info.get("input_cost_per_token")
         case "output_cost_per_token":
@@ -3821,7 +4067,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
             assert_never(key)
 
 
-def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
+def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
     """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
     because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
     if info is None:
@@ -3839,15 +4085,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
     )
 
 
-def _transcription_rate(
-    keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
-) -> float:
-    rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
-    return next((rate for rate in rates if rate is not None), 0.0)
+def _first_declared_transcription_rate(
+    keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
+) -> float | None:
+    rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
+    return next((rate for rate in rates if rate is not None), None)
+
+
+def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
+    return _first_declared_transcription_rate(keys, infos) or 0.0
 
 
 def _transcription_usage_cost(
-    usage: dict,
+    usage: Mapping[str, object],
     model_info: ModelInfo | None,
     override_info: ModelInfo | None = None,
 ) -> float:
@@ -3856,13 +4106,13 @@ def _transcription_usage_cost(
 
     usage_type: Final = usage.get("type")
     if usage_type == "duration":
-        seconds: Final = usage.get("seconds") or 0.0
+        seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
         return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
     if usage_type == "tokens":
-        input_token_details: Final = usage.get("input_token_details") or {}
-        audio_tokens: Final = input_token_details.get("audio_tokens") or 0
-        text_tokens: Final = input_token_details.get("text_tokens") or 0
-        output_tokens: Final = usage.get("output_tokens") or 0
+        input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
+        audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
+        text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
+        output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
         audio_cost: Final = float(audio_tokens) * _transcription_rate(
             ("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
         )

litellm/decisions/call.py

@@ -13,6 +13,7 @@ from typing_extensions import assert_never
 import litellm
 from litellm.cost_calculator import cost_map_key_prices_anything
 from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
+from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
 from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
@@ -555,7 +556,7 @@ def prepare_call(
         resolved_api_base, static_api_key = provider_config.resolve_connection(
             dynamic_api_base or call_api_base, dynamic_api_key or api_key
         )
-    except ValueError as error:
+    except (ValueError, httpx.InvalidURL) as error:
         raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
     if resolved_api_base is None:
         raise litellm.BadRequestError(
@@ -613,6 +614,9 @@ def prepare_call(
             litellm_params={
                 "litellm_call_id": kwargs.get("litellm_call_id"),
                 "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
+                "input_cost_per_query": (
+                    kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
+                ),
             },
             custom_llm_provider=provider,
         )
@@ -787,7 +791,9 @@ def _formatted_response(
         case DecisionsRequestBody():
             return ir_to_systemone_response(caller_response, caller_request)
         case OpenAIDecisionRequestBody():
-            return ir_to_openai_response(caller_response, caller_request, call.requested_model)
+            return ir_to_openai_response(
+                caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
+            )
         case _:
             assert_never(call.request)
 

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -113,7 +113,7 @@ class SpeechToCompletionBridgeHandler:
             voice=voice,
         )
 
-        with internal_sub_call():
+        with internal_sub_call(logging_obj.litellm_call_id):
             result: Final = completion(
                 **request_data,
             )

litellm/guardrails.py

@@ -0,0 +1,66 @@
+"""Invoke configured gateway guardrails without generating a completion."""
+
+from collections.abc import Mapping, Sequence
+from enum import Enum
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # shared factory has untyped optional parameters
+)
+from litellm.types.guardrails import NativeGuardrailResponse
+
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class _InputType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+async def apply(
+    guardrail_name: str,
+    *,
+    input: str | None = None,
+    response: str | None = None,
+    messages: Sequence[Mapping[str, object]] | None = None,
+    metadata: Mapping[str, object] | None = None,
+    api_base: str | None = None,
+    api_key: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+) -> NativeGuardrailResponse:
+    """Return a configured guardrail's native structured findings.
+
+    Supply exactly one of ``input`` or ``response``. Grounding guardrails can
+    use ``messages`` as evidence. Connection settings default to
+    ``litellm.api_base`` and ``litellm.api_key``.
+    """
+    if (input is None) == (response is None):
+        raise ValueError("Supply exactly one of input or response.")
+    resolved_base: Final = api_base if api_base is not None else litellm.api_base
+    if not resolved_base:
+        raise ValueError("Set api_base or litellm.api_base to your LiteLLM gateway URL.")
+    resolved_key: Final = api_key if api_key is not None else litellm.api_key
+    input_type: Final = _InputType.RESPONSE if response is not None else _InputType.REQUEST
+    payload: Final = dict[str, object](
+        guardrail_name=guardrail_name,
+        text=response if response is not None else input,
+        input_type=input_type.value,
+        response_format="native",
+    )
+    if messages is not None:
+        payload["messages"] = [dict(message) for message in messages]
+    if metadata is not None:
+        payload["metadata"] = dict(metadata)
+    gateway_root: Final = resolved_base.rstrip("/").removesuffix("/v1")
+    client: Final = get_async_httpx_client(llm_provider="litellm_guardrails")
+    result: Final = await client.post(  # pyright: ignore[reportUnknownMemberType]  # shared handler has untyped optional parameters
+        url=f"{gateway_root}/guardrails/apply_guardrail",
+        json=payload,
+        headers={"Authorization": f"Bearer {resolved_key}"} if resolved_key else {},
+        timeout=timeout,
+    )
+    return _NATIVE_RESPONSE.validate_json(result.content)

litellm/images/main.py

@@ -28,8 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
-from litellm.llms.custom_llm import CustomLLM
-from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
+from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params
 
 #################### Initialize provider clients ####################
 llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@@ -608,10 +607,7 @@ def image_generation(
             )
         elif custom_llm_provider in litellm._custom_providers:  # Assume custom LLM provider
             # Get the Custom Handler
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -915,10 +911,7 @@ def image_edit(
 
         # Check for custom provider
         if custom_llm_provider in litellm._custom_providers:
-            custom_handler: CustomLLM | None = None
-            for item in litellm.custom_provider_map:
-                if item["provider"] == custom_llm_provider:
-                    custom_handler = item["custom_handler"]
+            custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
 
             if custom_handler is None:
                 raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@@ -995,10 +988,15 @@ def image_edit(
             or custom_llm_provider == "azure"
             or custom_llm_provider in litellm.openai_compatible_providers
         ):
+            dropped_form_params: Final = header_auth_params(custom_llm_provider)
             image_edit_request_params.update(
                 flatten_form_field_values(
-                    non_default_params,
-                    extra_body if isinstance(extra_body, dict) else None,
+                    {key: value for key, value in non_default_params.items() if key not in dropped_form_params},
+                    (
+                        {key: value for key, value in extra_body.items() if key not in dropped_form_params}
+                        if isinstance(extra_body, dict)
+                        else None
+                    ),
                 )
             )
 

litellm/integrations/anthropic_cache_control_hook.py

@@ -9,29 +9,35 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
 
 """
 
+import bisect
 import copy
 import itertools
 import os
 import re
-from collections.abc import Callable, Container, Iterable, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
+from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
-from pydantic import TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import PromptManagementClient
-from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
     cache_control_as_prompt_cache_breakpoint,
+    prompt_cache_breakpoint_as_cache_control,
     with_prompt_cache_breakpoint,
+    without_prompt_cache_breakpoints,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     tool_call_ids_answered_by_client,
     tool_call_ids_answered_in,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
 from litellm.llms.anthropic.common_utils import (
     is_claude_code_one_shot_subagent_request,
     supports_anthropic_cache_control,
@@ -72,7 +78,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
 
 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
 OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
-_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
+_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+)|-(0|[1-9]\d?)(?![\da-z]))?")
 _FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
 _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
 OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
@@ -84,8 +90,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
 _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
 _CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object])
 _ATTRIBUTE_VALUE: Final = TypeAdapter(object)
+_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 
 AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
+CostMap: TypeAlias = Mapping[str, Mapping[str, object]]
+
+
+def routed_model_info(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MODEL_INFO_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap:
+    import litellm
+
+    if model_cost is not None:
+        return model_cost
+    return cast(CostMap, litellm.model_cost)  # cast-ok: the live map holds catalog entries
+
+
+def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None:
+    declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
+    return declared if isinstance(declared, bool) else None
+
+
+def routed_cost_map() -> CostMap:
+    from litellm.utils import catalog_with_runtime_registrations
+
+    return catalog_with_runtime_registrations()
+
+
+def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None:
+    return None if model_info is None else routed_cost_map()
 
 
 def _validated_object_mapping(value: object) -> dict[object, object] | None:
@@ -128,7 +166,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
         return False
     return (
         not isinstance(fields.get("content"), list)
-        or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
+        or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
     )
 
 
@@ -294,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object:
     return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
 
 
+def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object:
+    if not _carries_prompt_cache_breakpoint(part):
+        return part
+    return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params)
+
+
 def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
     if not isinstance(value, (list, tuple)):
         return ()
@@ -313,19 +357,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non
     return provider
 
 
-def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
-    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
+def supports_openai_prompt_cache_breakpoint(
+    model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool:
+    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True
 
 
-def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
+def openai_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
+) -> bool | None:
+    declared: Final = declared_prompt_cache_breakpoint(model_info)
+    if declared is not None:
+        return declared
+    model_cost: Final = _catalog_unless_unrouted(model_info)
     hosted_flag: Final = (
         None
         if custom_llm_provider is None
-        else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model)
+        else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost)
     )
     if hosted_flag is not None:
         return hosted_flag
-    model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
+    model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost)
     if model_map_flag is not None:
         return model_map_flag
     fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
@@ -334,6 +386,35 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str
     return _named_model_breakpoint_support(model)
 
 
+class _DeclaredBaseModels(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+
+    base_model: object = None
+    model_info: object = None
+
+    def names(self) -> tuple[str, ...]:
+        model_info: Final = _validated_object_mapping(self.model_info) or {}
+        candidates: Final = (self.base_model, model_info.get("base_model"))
+        return tuple(name for name in candidates if isinstance(name, str) and name)
+
+
+def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None:
+    return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info)
+
+
+def deployment_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str, litellm_params: object
+) -> bool | None:
+    """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
+    declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
+    model_info: Final = routed_model_info(declared.model_info)
+    supports: Final = (
+        openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info)
+        for name in dict.fromkeys((*declared.names(), model))
+    )
+    return next((support for support in supports if support is not None), None)
+
+
 def _named_model_breakpoint_support(model: str) -> bool | None:
     bare_model: Final = model.rsplit("/", 1)[-1].lower()
     if _O_SERIES_PATTERN.match(bare_model) is not None:
@@ -341,15 +422,14 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
     version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
     if version_match is None:
         return None
-    version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
+    version: Final = (int(version_match.group(1)), int(version_match.group(2) or version_match.group(3) or 0))
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 
 
-def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
-    import litellm
-
-    entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
-    flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
+def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None:
+    cost_map: Final = _live_cost_map_unless(model_cost)
+    entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1]))
+    flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping))
     return next((flag is True for flag in flags if flag is not None), None)
 
 
@@ -365,7 +445,7 @@ def _hosted_openai_dialect_flag(
     model: str,
     custom_llm_provider: str | None,
     resolve_provider: Callable[[str], str | None],
-    model_cost: Mapping[str, Mapping[str, object]] | None = None,
+    model_cost: CostMap | None = None,
 ) -> bool | None:
     """
     Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@@ -382,13 +462,7 @@ def _hosted_openai_dialect_flag(
     row of the same name, so an exact entry only speaks for a deployment when it is
     keyed for that deployment's provider.
     """
-    import litellm
-
-    cost_map: Final = (
-        cast(Mapping[str, Mapping[str, object]], litellm.model_cost)  # cast-ok: the live map holds catalog entries
-        if model_cost is None
-        else model_cost
-    )
+    cost_map: Final = _live_cost_map_unless(model_cost)
     exact_entry: Final = cost_map.get(model)
     exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
     if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
@@ -398,10 +472,16 @@ def _hosted_openai_dialect_flag(
         return None
     if isinstance(exact_entry, Mapping) and exact_provider == provider:
         return _hosted_entry_flag(exact_entry, lambda: provider)
+    entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider))
+    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
+    return next((flag for flag in flags if flag is not None), None)
+
+
+def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]:
     from litellm.utils import get_potential_model_names
 
     names: Final = get_potential_model_names(model, provider)
-    candidates: Final = (
+    return (
         names["combined_model_name"],
         names["region_free_combined_model_name"],
         names["split_model"],
@@ -409,9 +489,60 @@ def _hosted_openai_dialect_flag(
         names["stripped_model_name"],
         names["provider_prefixed_model_name"],
     )
-    entries: Final = (cost_map.get(candidate) for candidate in candidates)
-    flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
-    return next((flag for flag in flags if flag is not None), None)
+
+
+def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool:
+    entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider)))
+    return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries)
+
+
+_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse"))
+_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle"))
+
+
+def _declared_hosted_flag(
+    declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object
+) -> bool | None:
+    """
+    A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the
+    request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes
+    a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep
+    the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass
+    the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost
+    map, or the deployment's own ``litellm_provider``, lists under that provider.
+    """
+    if provider is None or provider == "openai":
+        return None
+    return (
+        declared
+        and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT
+        and (
+            provider in _PROVIDERS_HOSTING_OPENAIS_API
+            or deployment_provider == provider
+            or _cost_map_lists_model_for_provider(model, provider, model_cost)
+        )
+        and _sends_openai_shaped_chat(model, provider)
+    )
+
+
+@lru_cache(maxsize=512)
+def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool:
+    from litellm.exceptions import BadRequestError
+    from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
+    from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig
+    from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+    from litellm.llms.openai.openai import OpenAIConfig
+    from litellm.types.utils import LlmProviders
+    from litellm.utils import ProviderConfigManager
+
+    try:
+        served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider)
+        chat_config: Final = ProviderConfigManager.get_provider_chat_config(
+            model=served_model, provider=LlmProviders(provider)
+        )
+    except (BadRequestError, ValueError):
+        return False
+    return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig))
 
 
 def targets_openai_api(api_base: object) -> bool:
@@ -431,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool:
     return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS)
 
 
+def _carries_cache_control(block: object) -> bool:
+    return _attribute_or_key(block, "cache_control") is not None
+
+
 def _attribute_or_key(value: object, key: str) -> object | None:
     if hasattr(value, key):
         return _ATTRIBUTE_VALUE.validate_python(getattr(value, key))
@@ -445,22 +580,41 @@ def _as_object_list(value: object | None) -> list[object] | None:
     return _validated_object_list(value)
 
 
-def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, answered_tool_ids: Container[str]) -> bool:
+def _server_tool_call_id(tool_call: object) -> str | None:
+    tool_call_id: Final = _attribute_or_key(tool_call, "id")
+    if isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
+        return tool_call_id
+    return None
+
+
+def _tool_call_carries_cache_breakpoint(tool_call: object, client_tool_call_ids: Container[str]) -> bool:
     if _attribute_or_key(tool_call, "cache_control") is None:
         return False
+    server_tool_call_id: Final = _server_tool_call_id(tool_call)
+    return server_tool_call_id is None or server_tool_call_id in client_tool_call_ids
 
-    return not tool_call_is_rebuilt_as_server_tool_use(
-        _attribute_or_key(tool_call, "id"), _attribute_or_key(message, "provider_specific_fields"), answered_tool_ids
+
+def _server_tool_call_ids_without_a_kept_result(message: object) -> Iterator[str]:
+    provider_specific_fields: Final = _attribute_or_key(message, "provider_specific_fields")
+    tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or ()
+    return (
+        tool_call_id
+        for tool_call in tool_calls
+        if (tool_call_id := _server_tool_call_id(tool_call)) is not None
+        and not tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
     )
 
 
-def _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+def _client_tool_call_ids_when_a_server_tool_call_is_marked(
     messages: Sequence[object], model: str | None = None, custom_llm_provider: str | None = None
 ) -> frozenset[str]:
     if not _marks_a_server_tool_call(messages):
         return frozenset()
     if model is not None and not _builds_anthropic_messages(model, custom_llm_provider):
-        return tool_call_ids_answered_in(messages)
+        server_calls_sent_as_client_calls: Final = itertools.chain.from_iterable(
+            map(_server_tool_call_ids_without_a_kept_result, messages)
+        )
+        return tool_call_ids_answered_in(messages) | frozenset(server_calls_sent_as_client_calls)
     return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider))
 
 
@@ -491,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b
     return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig)
 
 
+def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
+    return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages(
+        model, custom_llm_provider
+    )
+
+
 def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
     provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
     return provider if isinstance(provider, str) else None
@@ -517,12 +677,7 @@ def _model_info_flags_mid_conversation_system(model: str, custom_llm_provider: s
 
 
 def _is_marked_server_tool_call(tool_call: object) -> bool:
-    tool_call_id: Final = _attribute_or_key(tool_call, "id")
-    return (
-        isinstance(tool_call_id, str)
-        and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
-        and _attribute_or_key(tool_call, "cache_control") is not None
-    )
+    return _server_tool_call_id(tool_call) is not None and _attribute_or_key(tool_call, "cache_control") is not None
 
 
 def _tool_carries_cache_breakpoint(tool: object) -> bool:
@@ -539,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
     return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
 
 
+def _carries_prompt_cache_breakpoint(block: object) -> bool:
+    return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
+
+
 def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
     return message.get("role") != "assistant"
 
 
+def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
+    return message.get("role") == "tool" and any(
+        _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
+    )
+
+
+def _part_without_prompt_cache_breakpoint(part: object) -> object:
+    if not _carries_prompt_cache_breakpoint(part):
+        return part
+    return {
+        key: value
+        for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items()
+        if key != "prompt_cache_breakpoint"
+    }
+
+
+def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
+    if not _tool_result_carries_prompt_cache_breakpoint(message):
+        return message
+    parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
+    return cast(  # cast-ok: the same tool message with its part marks removed
+        AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]}
+    )
+
+
+def _created_cache_control(original: object, translated: object) -> bool:
+    return (
+        _attribute_or_key(original, "cache_control") is None
+        and _attribute_or_key(translated, "cache_control") is not None
+    )
+
+
+def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]:
+    if translated is original:
+        return ()
+    original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or []
+    translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or []
+    return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts))
+
+
+def _part_without_cache_control(part: object) -> object:
+    return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"}
+
+
+def _message_dropping_created_cache_controls_between(
+    message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int
+) -> AllMessageValues:
+    if not any(created):
+        return message
+    parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
+    ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before)
+    return cast(  # cast-ok: the same message with its middle translated marks removed
+        AllMessageValues,
+        {
+            **message,
+            "content": [
+                _part_without_cache_control(part)
+                if is_created and first_dropped <= ordinal < first_kept_after
+                else part
+                for part, is_created, ordinal in zip(parts, created, ordinals)
+            ],
+        },
+    )
+
+
 # Set by a caller whose message list is not the one that goes upstream -- today the
 # Responses API layer, whose `instructions` only becomes a system message further down.
 # Tells this hook to hand role-targeted points to the pass holding the final messages
@@ -628,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if not injection_points:
             return model, messages, non_default_params
 
-        # Create a deep copy of messages to avoid modifying the original list
-        processed_messages = copy.deepcopy(messages)
-
         message_points: Final = tuple(
             cast(CacheControlMessageInjectionPoint, point)
             for point in injection_points
@@ -638,18 +859,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message")
 
-        custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0])
+        request_params: Final = cast(Mapping[str, object], non_default_params)  # cast-ok: base hook's bare dict
+        custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
         stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
+        model_info: Final = routed_model_info(request_params.get("model_info"))
         openai_dialect: Final = (
             stamped_dialect
             if isinstance(stamped_dialect, bool)
             else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
                 model,
                 custom_llm_provider,
-                non_default_params.get("api_base") or non_default_params.get("base_url"),
-                non_default_params.get("prompt_cache_options"),
+                request_params.get("api_base") or request_params.get("base_url"),
+                request_params.get("prompt_cache_options"),
+                model_info,
             )
         )
+        processed_messages = copy.deepcopy(
+            [without_prompt_cache_breakpoints(message) for message in messages]
+            if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
+            else messages
+        )
         # A provisional message list defers every role-targeted point to the pass holding
         # the final one: a role with no message here may have one there, and settling all
         # of them in one pass is what lets config order decide the shared breakpoint
@@ -668,7 +897,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
             remaining_points, external_breakpoints, openai_dialect
         )
-        answered_tool_ids: Final = _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+        answered_tool_ids: Final = _client_tool_call_ids_when_a_server_tool_call_is_marked(
             processed_messages, model, custom_llm_provider
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(
@@ -718,32 +947,35 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object = None,
         prompt_cache_options: object = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> bool:
         if model is None:
             return False
-        hosted_flag: Final = _hosted_openai_dialect_flag(
-            model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
+        declared: Final = declared_prompt_cache_breakpoint(model_info)
+        hosted_flag: Final = (
+            _hosted_openai_dialect_flag(
+                model,
+                custom_llm_provider,
+                lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base),
+                _catalog_unless_unrouted(model_info),
+            )
+            if declared is None
+            else _declared_hosted_flag(
+                declared,
+                model,
+                custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base),
+                routed_cost_map(),
+                None if model_info is None else model_info.get("litellm_provider"),
+            )
         )
         if hosted_flag is not None:
-            return hosted_flag
-        if not supports_openai_prompt_cache_breakpoint(model):
+            return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
+        if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
             return False
         if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
             return False
         return prompt_cache_options is not None or targets_openai_api(api_base)
 
-    @staticmethod
-    def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
-        return (
-            _hosted_openai_dialect_flag(
-                model,
-                custom_llm_provider,
-                AnthropicCacheControlHook._resolve_provider,
-                GetModelCostMap.loaded_model_cost_map(),
-            )
-            is True
-        )
-
     @staticmethod
     def _resolve_provider(model: str, api_base: object = None) -> str | None:
         return _served_provider_for_model(model, api_base)
@@ -757,7 +989,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         message_list: Final = tuple(messages)
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(message_list)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(message_list)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -790,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
 
+    @staticmethod
+    def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]:
+        return tuple(
+            tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool)
+        )
+
+    @staticmethod
+    def count_chat_external_cache_breakpoints(optional_params: object) -> int:
+        params: Final = _validated_object_mapping(optional_params) or {}
+        return AnthropicCacheControlHook.count_external_cache_breakpoints(
+            _as_object_list(params.get("tools")), params.get("cache_control")
+        )
+
     @staticmethod
     def count_external_cache_breakpoints_on_messages_route(
         tools: Iterable[object] | None, cache_control: object, request_kwargs: object
@@ -849,7 +1094,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         so earlier points win when slots are scarce.
         """
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(messages)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -939,16 +1184,22 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
         tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
         tool_call_count: Final = (
-            sum(
-                1
-                for tool_call in tool_calls
-                if _tool_call_carries_cache_breakpoint(tool_call, message, answered_tool_ids)
-            )
+            sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, answered_tool_ids))
             if tool_calls
             else 0
         )
         return message_count + content_count + tool_call_count
 
+    @staticmethod
+    def count_converse_cache_points(message: AllMessageValues) -> int:
+        content: Final = _as_object_list(_attribute_or_key(message, "content"))
+        tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or []
+        marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block))
+        marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call))
+        if message.get("role") == "tool":
+            return int(_carries_cache_control(message) or marked_parts > 0)
+        return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls
+
     @staticmethod
     def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool:
         """Return True if the message already carries any cache_control."""
@@ -1007,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             return renamed
         return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
 
+    @staticmethod
+    def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
+        """Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's
+        ``cache_control``. Marks with no OpenAI slot are left for the caller to strip."""
+        if not _may_carry_prompt_cache_breakpoint(message):
+            return message
+        parts: Final = _as_object_list(_attribute_or_key(message, "content"))
+        if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts):
+            return message
+        return cast(  # cast-ok: the same message with its part marks renamed
+            AllMessageValues,
+            {**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]},
+        )
+
+    @staticmethod
+    def prompt_cache_breakpoints_dropped_from_tool_results(
+        messages: Sequence[AllMessageValues], drop_params: bool, model: str
+    ) -> tuple[AllMessageValues, ...]:
+        if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages):
+            return tuple(messages)
+        if not drop_params:
+            raise UnsupportedParamsError(
+                message=(
+                    f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool "
+                    f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
+                ),
+                status_code=400,
+            )
+        verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model)
+        return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages)
+
+    @staticmethod
+    def prompt_cache_breakpoints_as_capped_cache_control(
+        messages: Sequence[AllMessageValues],
+        drop_params: bool,
+        sent_breakpoints: Callable[[AllMessageValues], int],
+        external_breakpoints: Callable[[], int],
+    ) -> tuple[AllMessageValues, ...]:
+        """``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's
+        breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends.
+        Under ``drop_params`` the first and last translated marks fill the slots the client's own
+        ``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest
+        turns stay cached, and without it a request over the cap is rejected."""
+        translated: Final = tuple(
+            AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params)
+            for message in messages
+        )
+        if all(after is before for before, after in zip(messages, translated)):
+            return translated
+        created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated))
+        created_marks: Final = sum(sum(flags) for flags in created)
+        if created_marks == 0:
+            return translated
+        external: Final = external_breakpoints()
+        breakpoints: Final = sum(map(sent_breakpoints, translated)) + external
+        excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS
+        if excess <= 0:
+            return translated
+        if not drop_params:
+            raise UnsupportedParamsError(
+                message=(
+                    f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this "
+                    f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} "
+                    f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them."
+                ),
+                status_code=400,
+            )
+        verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params")
+        marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0))
+
+        def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]:
+            first_dropped: Final = (kept_marks + 1) // 2
+            first_kept_after: Final = created_marks - kept_marks // 2
+            return tuple(
+                _message_dropping_created_cache_controls_between(
+                    message, flags, before, first_dropped, first_kept_after
+                )
+                for message, flags, before in zip(translated, created, marks_before)
+            )
+
+        def over_cap(kept_marks: int) -> bool:
+            return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS
+
+        first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap)
+        return capped_request(max(0, first_over_cap - 1))
+
     @staticmethod
     def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
         if not _may_carry_prompt_cache_breakpoint(message):
@@ -1146,6 +1483,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object,
         prompt_cache_options: object,
+        model_info: Mapping[str, object] | None = None,
     ) -> Sequence[Mapping[str, object]]:
         """Carry onto the points what the prompt-management hook never receives.
 
@@ -1157,7 +1495,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         across requests.
         """
         with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect(
-            points, model, custom_llm_provider, api_base, prompt_cache_options
+            points, model, custom_llm_provider, api_base, prompt_cache_options, model_info
         )
         stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect)
         if external_breakpoints == 0 and not stamped_earlier:
@@ -1171,14 +1509,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         custom_llm_provider: str | None,
         api_base: object,
         prompt_cache_options: object,
+        model_info: Mapping[str, object] | None = None,
     ) -> Sequence[Mapping[str, object]]:
-        if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider):
+        if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(
+            model, custom_llm_provider
+        ):
             return points
         return AnthropicCacheControlHook._stamped(
             points,
             "_litellm_openai_dialect",
             AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-                model, custom_llm_provider, api_base, prompt_cache_options
+                model, custom_llm_provider, api_base, prompt_cache_options, model_info
             ),
         )
 
@@ -1237,6 +1578,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         cache_control: object = None,
         request_kwargs: object = None,
         on_messages_route: bool = False,
+        model_info: Mapping[str, object] | None = None,
     ) -> list[CacheControlInjectionPoint]:
         """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
 
@@ -1254,7 +1596,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True:
             return []
 
-        if not supports_anthropic_cache_control(model, custom_llm_provider):
+        if not supports_anthropic_cache_control(model, custom_llm_provider, model_info):
             return []
 
         if AnthropicCacheControlHook._request_has_cache_control(
@@ -1332,6 +1674,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         tools: list | None = None,
         enable_prompt_caching: bool | None = None,
         api_base: object = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> None:
         """For /chat/completions: resolve the injection points the request should carry.
 
@@ -1346,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
 
         configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points"))
         if configured:
-            tools_keeping_marks: Final = tuple(
-                tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool)
-            )
+            tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools)
             stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook(
                 configured,
                 AnthropicCacheControlHook.count_external_cache_breakpoints(
@@ -1358,6 +1699,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 custom_llm_provider,
                 api_base,
                 non_default_params.get("prompt_cache_options"),
+                model_info,
             )
             non_default_params["cache_control_injection_points"] = (
                 AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider)
@@ -1374,6 +1716,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             enable_prompt_caching=enable_prompt_caching,
             cache_control=non_default_params.get("cache_control"),
             request_kwargs=non_default_params,
+            model_info=model_info,
         )
         if points:
             non_default_params["cache_control_injection_points"] = points
@@ -1492,6 +1835,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 cache_control=cache_control,
                 request_kwargs=kwargs,
                 on_messages_route=True,
+                model_info=routed_model_info(kwargs.get("model_info")),
             )
             if model is not None
             else ()
@@ -1500,7 +1844,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             return messages, system
 
         openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-            model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
+            model,
+            custom_llm_provider,
+            api_base,
+            kwargs.get("prompt_cache_options"),
+            routed_model_info(kwargs.get("model_info")),
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
         messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
@@ -1634,9 +1982,17 @@ def messages_within_cache_control_cap(
 
     Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
     """
-    reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
-        _validated_object_list(tools), cache_control, request_kwargs
+    return messages_within_reserved_cache_control_cap(
+        messages,
+        AnthropicCacheControlHook.count_external_cache_breakpoints(
+            _validated_object_list(tools), cache_control, request_kwargs
+        ),
     )
+
+
+def messages_within_reserved_cache_control_cap(
+    messages: Sequence[_MessageT], reserved_marks: int
+) -> Sequence[_MessageT]:
     kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
     dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
     if dropped > 0:

litellm/integrations/azure_storage/azure_storage.py

@@ -1,7 +1,8 @@
 import asyncio
 import os
 import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from functools import cache
 from typing import Final
@@ -15,6 +16,7 @@ from litellm.constants import (
     AZURE_STORAGE_MSFT_VERSION,
 )
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
 from litellm.llms.custom_httpx.http_handler import (
@@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
     return f"{name.translate(_ADLS_SAFE_NAME)}.json"
 
 
+@dataclass(frozen=True, slots=True)
+class AzureStorageLogQueueItem:
+    payload: StandardLoggingPayload
+    file_name: str
+
+
+def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
+    payload_id: Final = payload.get("id")
+    return AzureStorageLogQueueItem(
+        payload=payload,
+        file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
+    )
+
+
 @cache
 def _cached_credential_chain_token_provider() -> Callable[[], str]:
     return get_azure_ad_token_provider(
@@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
 
             asyncio.create_task(self.periodic_flush())
             self.flush_lock = asyncio.Lock()
-            self.log_queue: list[StandardLoggingPayload] = []
+            self.log_queue: list[AzureStorageLogQueueItem] = []
             super().__init__(**kwargs, flush_lock=self.flush_lock)
         except Exception as e:
             verbose_logger.exception(
@@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
     def azure_storage_blob_endpoint(self) -> str:
         return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
 
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
         """
         Async Log success events to Azure Blob Storage
 
@@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             if standard_logging_payload is None:
                 raise ValueError("standard_logging_payload is not set")
 
-            self.log_queue.append(standard_logging_payload)
+            self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
 
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
 
-    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
         """
         Async Log failure events to Azure Blob Storage
 
@@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             if standard_logging_payload is None:
                 raise ValueError("standard_logging_payload is not set")
 
-            self.log_queue.append(standard_logging_payload)
+            self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
 
@@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
                 len(self.log_queue),
             )
 
-            for payload in self.log_queue:
-                await self.async_upload_payload_to_azure_blob_storage(payload=payload)
+            for item in self.log_queue:
+                await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
 
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
 
-    async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
+    async def async_upload_payload_to_azure_blob_storage(
+        self, payload: StandardLoggingPayload, file_name: str | None = None
+    ):
         """
         Uploads the payload to Azure Blob Storage using a 3-step process:
         1. Create file resource
@@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
         """
         try:
             if self.azure_storage_account_key:
-                await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
+                await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
             else:
                 # Get a valid token instead of always requesting a new one
                 await self.set_valid_azure_ad_token()
                 async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
                 json_payload: Final = safe_dumps(payload) + "\n"  # Add newline for each log entry
                 payload_bytes: Final = json_payload.encode("utf-8")
-                filename: Final = adls_safe_file_name(payload.get("id"))
+                filename: Final = file_name or adls_safe_file_name(payload.get("id"))
                 base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
 
                 # Execute the 3-step upload process
@@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
         return self._service_client
 
-    async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
+    async def upload_to_azure_data_lake_with_azure_account_key(
+        self, payload: StandardLoggingPayload, file_name: str | None = None
+    ):
         """
         Uploads the payload to Azure Data Lake using the Azure SDK
 
@@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
 
         try:
             today: Final = datetime.now().strftime("%Y-%m-%d")
-            file_name: Final = adls_safe_file_name(payload.get("id"))
+            resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
             await self._upload_bytes_with_account_key(
-                file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
+                file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
             )
-            verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
+            verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
 
         except Exception as e:
             verbose_logger.exception("Error occurred: %s", e)

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
 from litellm.integrations.clickhouse.context import is_lens_analysis
 from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@@ -134,7 +135,7 @@ def spend_log_row_from_payload(
     usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
     cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
     trace_id, span_id = _find_traceparent(metadata, kwargs)
-    request_id = str(payload.get("id") or "")
+    response_id: Final = str(payload.get("id") or "")
     redact = litellm.turn_off_message_logging is True
     track_end_user: Final = not litellm.disable_end_user_cost_tracking
     completion_start_ms = _to_ms(payload.get("completionStartTime"))
@@ -157,8 +158,8 @@ def spend_log_row_from_payload(
         )
     )
     return SpendLogRecord(
-        request_id=request_id,
-        response_id=strip_cache_hit_suffix(request_id),
+        request_id=logged_event_key(kwargs, response_id),
+        response_id=strip_cache_hit_suffix(response_id),
         provider_request_id=(
             get_provider_request_id(kwargs.get("response_headers"))
             or get_provider_request_id(hidden_params.get("additional_headers"))

litellm/integrations/custom_guardrail.py

@@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
         enrich_http_exception_with_guardrail_context(result, self)
         raise result
 
+    def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
+        return (
+            isinstance(request_data.get("guardrails"), list)
+            and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
+        )
+
     async def async_post_call_success_deployment_hook(
         self,
         request_data: dict,
@@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
         """
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
-        # should run guardrail
-        litellm_guardrails: Final = request_data.get("guardrails")
-        if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
-            return None
-
-        if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
+        if not self.runs_post_call_deployment_hook(request_data):
             return None
 
         target: Final = self._deployment_hook_target()

litellm/integrations/deepeval/deepeval.py

@@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
     to_zod_compatible_iso,
     validate_environment,
 )
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 
 
 # This file includes the custom callbacks for LiteLLM Proxy
@@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
             )
         else:
             output = str(standard_logging_object.get("error_string", ""))
+        logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
+        span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
         return BaseApiSpan(
-            uuid=standard_logging_object.get("id", uuid.uuid4()),
+            uuid=span_id,
             name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
             status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
             type=SpanApiType.LLM,

litellm/integrations/dynamodb.py

@@ -9,6 +9,8 @@ from typing import Final, Protocol
 
 import litellm
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 
 
 class _DynamoTable(Protocol):
@@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
 class DyanmoDBLogger:
     # Class variables or attributes
 
-    def __init__(self):
+    def __init__(self, dynamodb: _DynamoResource | None = None):
         # Instance variables
         import boto3
 
-        self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
+        self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
+            "dynamodb", region_name=os.environ["AWS_REGION_NAME"]
+        )
         if litellm.dynamodb_table_name is None:
             raise ValueError(
                 "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@@ -45,23 +49,23 @@ class DyanmoDBLogger:
 
     _async_log_event = async_log_event
 
-    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+    def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
         try:
             print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
 
             # construct payload to send to DynamoDB
             # follows the same params as langfuse.py
-            litellm_params: Final = kwargs.get("litellm_params", {})
-            metadata: Final = litellm_params.get("metadata", {}) or {}  # if litellm_params['metadata'] == None
+            litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+            metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
             messages: Final = kwargs.get("messages")
             optional_params: Final = kwargs.get("optional_params", {})
             call_type: Final = kwargs.get("call_type", "litellm.completion")
             usage: Final = response_obj["usage"]
-            id: Final = response_obj.get("id", str(uuid.uuid4()))
+            response_id: Final = response_obj.get("id", str(uuid.uuid4()))
 
             # Build the initial payload
             payload: Final[dict[str, object]] = {
-                "id": id,
+                "id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
                 "call_type": call_type,
                 "startTime": start_time,
                 "endTime": end_time,

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
     sanitize_cloud_object_component,
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
 from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         else:
             await self._send_individual_logs(items_to_process)
 
-    def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
+    def _get_object_name(
+        self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
+    ) -> str:
         """
         Get the object name to use for the current payload
         """
         current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
+        metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
+        gcs_log_id: Final = metadata.get("gcs_log_id")
+        safe_log_id: Final = sanitize_cloud_object_component(
+            gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
+        )
+        if safe_log_id:
+            return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
         if logging_payload.get("error_str", None) is not None:
-            object_name = self._generate_failure_object_name(
-                request_date_str=current_date,
-            )
-        else:
-            object_name = self._generate_success_object_name(
-                request_date_str=current_date,
-                response_id=response_obj.get("id", ""),
-            )
-
-        # used for testing
-        _litellm_params: Final = kwargs.get("litellm_params", None) or {}
-        _metadata: Final = _litellm_params.get("metadata", None) or {}
-        if "gcs_log_id" in _metadata:
-            safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
-            if safe_log_id:
-                object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
-
-        return object_name
+            return self._generate_failure_object_name(request_date_str=current_date)
+        raw_response_id: Final[object] = response_obj.get("id", "")
+        response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
+        return self._generate_success_object_name(
+            request_date_str=current_date,
+            response_id=logged_event_key(kwargs, response_id),
+        )
 
     async def get_request_response_payload(
         self,

litellm/integrations/openmeter.py

@@ -3,12 +3,15 @@
 
 import json
 import os
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
     get_async_httpx_client,
@@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
         if len(missing_keys) > 0:
             raise Exception(f"Missing keys={missing_keys} in environment.")
 
-    def _common_logic(self, kwargs: dict, response_obj):
+    def _common_logic(self, kwargs: Mapping[str, object], response_obj):
         call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
         dt: Final = get_utc_datetime().isoformat()
         cost: Final = kwargs.get("response_cost", None)
@@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
         # If no user provided directly, try to get it from token user_id
         if user_param is None:
             # Check if user_id is available from the API key metadata
-            litellm_params: Final = kwargs.get("litellm_params", {})
-            metadata: Final = litellm_params.get("metadata", {})
+            litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+            metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
             user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
 
             if user_api_key_user_id is not None:
@@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
         return {
             "specversion": "1.0",
             "type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
-            "id": call_id,
+            "id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
             "time": dt,
             "subject": subject,
             "source": "litellm-proxy",

litellm/integrations/prometheus.py

@@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
 
     async def async_post_call_failure_hook(
         self,
-        request_data: dict,
+        request_data: dict[str, object],
         original_exception: Exception,
         user_api_key_dict: UserAPIKeyAuth,
         traceback_str: str | None = None,
@@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
         try:
             _tags: Final = StandardLoggingPayloadSetup.get_request_tags(
                 litellm_params=request_data,
-                proxy_server_request=request_data.get("proxy_server_request", {}),
+                proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
+            )
+            _metadata: Final = _logged_fields(request_data.get("metadata"))
+            model_id: Final = (
+                _logged_fields(_metadata.get("model_info")).get("id")
+                or _logged_fields(request_data.get("model_info")).get("id")
+                or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
             )
-            _metadata: Final = request_data.get("metadata", {}) or {}
-            model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
             rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
             api_provider: Final = self._extract_api_provider_from_request_data(
                 request_data

litellm/integrations/websearch_interception/handler.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
 from typing_extensions import Never, ReadOnly
 
 import litellm
+from litellm._internal_context import billed_separately
 from litellm._logging import verbose_logger
 from litellm.anthropic_interface import messages as anthropic_messages
 from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
             search_kwargs: Final = MappingProxyType(
                 {**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
             )
-            result: Final = (
-                await litellm.asearch(
-                    query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+            with billed_separately():
+                result: Final = (
+                    await litellm.asearch(
+                        query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+                    )
+                    if search_metadata is None
+                    else await litellm.asearch(
+                        query=query_arg,
+                        search_provider=search_provider,
+                        litellm_metadata=search_metadata,
+                        **_NO_ASEARCH_NAMED,
+                        **search_kwargs,
+                    )
                 )
-                if search_metadata is None
-                else await litellm.asearch(
-                    query=query_arg,
-                    search_provider=search_provider,
-                    litellm_metadata=search_metadata,
-                    **_NO_ASEARCH_NAMED,
-                    **search_kwargs,
-                )
-            )
 
             # Format using transformation function
             search_result_text: Final = WebSearchTransformation.format_search_response(result)

litellm/litellm_core_utils/allowed_tools_tool_choice.py

@@ -0,0 +1,111 @@
+from collections.abc import Mapping
+from typing import Final, Literal
+
+from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+from litellm.exceptions import BadRequestError
+
+ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
+
+CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
+    '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
+)
+
+
+class ChatAllowedTools(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    mode: Literal["auto", "required"]
+    tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
+
+
+class ChatAllowedToolsToolChoice(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    type: Literal["allowed_tools"]
+    allowed_tools: ChatAllowedTools
+
+
+_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
+
+
+_SELECTOR: Final = TypeAdapter(dict[str, object])
+
+
+class _ToolChoiceType(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str
+
+
+_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType)
+
+
+def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
+    try:
+        return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools"
+    except ValidationError:
+        return False
+
+
+def raise_if_allowed_tools_tool_choice_unsupported(
+    tool_choice: object,
+    custom_llm_provider: str,
+    model: str,
+) -> None:
+    """Providers outside the set would silently send the request with no tool restriction at all, so the call stops
+    here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
+    if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
+        return
+    raise BadRequestError(
+        message=(
+            "tool_choice type 'allowed_tools' is only supported for "
+            f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
+            "drop_params does not drop it, since that would let the model call tools the request excluded. "
+            "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
+        ),
+        model=model,
+        llm_provider=custom_llm_provider,
+    )
+
+
+def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
+    try:
+        return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
+    except ValidationError:
+        return None
+
+
+def _has_name(selector: Mapping[str, object]) -> bool:
+    name: Final = selector.get("name")
+    return isinstance(name, str) and name != ""
+
+
+def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
+    tool_type: Final = tool.get("type")
+    if tool_type not in ("function", "custom"):
+        return tool
+    if tool_type not in tool:
+        return tool if _has_name(tool) else None
+    try:
+        nested: Final = _SELECTOR.validate_python(tool[tool_type])
+    except ValidationError:
+        return None
+    outer: Final = {key: value for key, value in tool.items() if key != tool_type}
+    if not _has_name(nested) or outer.keys() & nested.keys():
+        return None
+    return {**outer, **nested}
+
+
+def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None:
+    """The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is malformed or
+    carries conflicting keys, so the caller sends the original rather than a rewrite that could mean something looser."""
+    flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools)
+    if any(tool is None for tool in flattened):
+        return None
+    return ToolChoiceAllowedParam(
+        type="allowed_tools",
+        mode=choice.allowed_tools.mode,
+        tools=[dict(tool) for tool in flattened if tool is not None],
+    )

litellm/litellm_core_utils/audio_utils/transcription_streaming.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import datetime
+import json
+import traceback
+from collections.abc import AsyncIterator, Callable, Iterator, Mapping
+from typing import Final, Protocol
+
+import anyio
+from openai import AsyncStream, Stream
+from openai.types.audio import (
+    TranscriptionStreamEvent,
+    TranscriptionTextDeltaEvent,
+    TranscriptionTextDoneEvent,
+)
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    TranscriptionUsageObjectTransformation,
+)
+from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
+
+_STREAM_FLAG: Final = TypeAdapter(bool)
+
+
+def transcription_stream_flag(value: object) -> bool | None:
+    try:
+        return _STREAM_FLAG.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def transcription_stream_unset(value: object) -> bool:
+    return value is None or value == ""
+
+
+def transcription_stream_requested(value: object) -> bool:
+    return transcription_stream_flag(value) is True
+
+
+TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
+
+TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
+
+
+def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
+    from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
+
+    model: Final = call_details.get("model")
+    custom_llm_provider: Final = call_details.get("custom_llm_provider")
+    extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
+
+    def map_exception(exception: Exception) -> Exception:
+        try:
+            mapped: Final[object] = exception_type(
+                model=model if isinstance(model, str) else None,
+                original_exception=exception,
+                custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+                extra_kwargs=extra_kwargs,
+            )
+        except Exception as mapped_exception:  # noqa: BLE001  # exception_type raises the mapped exception
+            return mapped_exception
+        return mapped if isinstance(mapped, Exception) else exception
+
+    return map_exception
+
+
+def _guarded_transcription_event_payloads(
+    response: TranscriptionResponse,
+) -> tuple[dict[str, object], dict[str, object]]:
+    text: Final = response.text or ""
+    usage: Final = response.usage
+    languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
+    return (
+        {"type": "transcript.text.delta", "delta": text},
+        {
+            "type": "transcript.text.done",
+            "text": text,
+            **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
+            **({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
+        },
+    )
+
+
+def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
+    return (
+        *(
+            f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
+            for event in _guarded_transcription_event_payloads(response)
+        ),
+        TRANSCRIPTION_STREAM_DONE_FRAME,
+    )
+
+
+class GuardedTranscriptionReplay:
+    def __init__(self, response: TranscriptionResponse) -> None:
+        self.response: Final = response
+        self._hidden_params: dict[str, object] = response.hidden_params
+
+    def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
+        return self._events()
+
+    async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
+        delta, done = _guarded_transcription_event_payloads(self.response)
+        yield TranscriptionTextDeltaEvent.model_construct(**delta)
+        yield TranscriptionTextDoneEvent.model_construct(**done)
+
+
+class TranscriptionStreamLogging(Protocol):
+    def success_handler(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    async def async_success_handler(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def handle_sync_success_callbacks_for_async_calls(
+        self,
+        result: TranscriptionResponse,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def failure_handler(
+        self,
+        exception: Exception,
+        traceback_exception: str,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+        *,
+        for_async_request: bool = False,
+    ) -> None: ...
+
+    async def async_failure_handler(
+        self,
+        exception: Exception,
+        traceback_exception: str,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> None: ...
+
+    def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
+
+    def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
+
+
+class _TranscriptionEventCollector:
+    def __init__(self, duration: float | None) -> None:
+        self.duration = duration
+        self.text_deltas: list[str] = []  # mutable-ok: streaming deltas accumulate until the terminal event
+        self.done_event: TranscriptionTextDoneEvent | None = None
+
+    def add(self, event: TranscriptionStreamEvent) -> None:
+        if isinstance(event, TranscriptionTextDeltaEvent):
+            self.text_deltas.append(event.delta)
+        elif isinstance(event, TranscriptionTextDoneEvent):
+            self.done_event = event
+
+    def response(self) -> TranscriptionResponse:
+        done_event: Final = self.done_event
+        response: Final = TranscriptionResponse(
+            text=done_event.text if done_event is not None else "".join(self.text_deltas),
+            usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
+            languages=getattr(done_event, "languages", None) if done_event is not None else None,
+        )
+        if self.duration is not None:
+            response.set_audio_transcription_duration(self.duration)
+        return response
+
+
+def _billed_usage(response: TranscriptionResponse) -> Usage:
+    usage: Final[object] = getattr(response, "usage", None)
+    token_usage: Final = (
+        TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
+        if isinstance(usage, TranscriptionUsageTokensObject)
+        else None
+    )
+    return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+
+
+def _record_billable_partial_spend(
+    logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
+) -> None:
+    partial_response: Final = collector.response()
+    partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
+    if partial_cost <= 0:
+        return
+    logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
+
+
+class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
+    def __init__(
+        self,
+        stream: Stream[TranscriptionStreamEvent],
+        logging_obj: TranscriptionStreamLogging,
+        start_time: datetime.datetime,
+        map_exception: TranscriptionStreamExceptionMapper,
+    ) -> None:
+        self.__dict__.update(stream.__dict__)
+        self._hidden_params: dict[str, object] = {  # mutable-ok: ResponseMetadata fills in the call's metadata
+            "additional_headers": process_response_headers(stream.response.headers)
+        }
+        self._logging_obj = logging_obj
+        self._start_time = start_time
+        self._map_exception = map_exception
+        self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
+        self._finalized = False
+        self._failed = False
+        source_iterator: Final = self._iterator
+        self._iterator = self._logging_iterator(source_iterator)
+
+    def _logging_iterator(
+        self, source_iterator: Iterator[TranscriptionStreamEvent]
+    ) -> Iterator[TranscriptionStreamEvent]:
+        try:
+            for event in source_iterator:
+                self._collector.add(event)
+                yield event
+        except Exception as exception:
+            self._failed = True
+            _record_billable_partial_spend(self._logging_obj, self._collector)
+            mapped_exception: Final = self._map_exception(exception)
+            end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+            self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
+            raise mapped_exception from exception
+        finally:
+            self._finalize()
+
+    def _finalize(self) -> None:
+        if self._finalized or self._failed:
+            return
+        self._finalized = True
+        self._logging_obj.success_handler(
+            self._collector.response(),
+            self._start_time,
+            datetime.datetime.now(),  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+        )
+
+    def close(self) -> None:
+        try:
+            super().close()
+        finally:
+            self._finalize()
+
+    @property
+    def logging_obj(self) -> TranscriptionStreamLogging:
+        return self._logging_obj
+
+
+class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
+    def __init__(
+        self,
+        stream: AsyncStream[TranscriptionStreamEvent],
+        logging_obj: TranscriptionStreamLogging,
+        start_time: datetime.datetime,
+        map_exception: TranscriptionStreamExceptionMapper,
+    ) -> None:
+        self.__dict__.update(stream.__dict__)
+        self._hidden_params: dict[str, object] = {  # mutable-ok: ResponseMetadata fills in the call's metadata
+            "additional_headers": process_response_headers(stream.response.headers)
+        }
+        self._logging_obj = logging_obj
+        self._start_time = start_time
+        self._map_exception = map_exception
+        self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
+        self._finalized = False
+        self._failed = False
+        source_iterator: Final = self._iterator
+        self._iterator = self._logging_iterator(source_iterator)
+
+    async def _logging_iterator(
+        self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
+    ) -> AsyncIterator[TranscriptionStreamEvent]:
+        try:
+            async for event in source_iterator:
+                self._collector.add(event)
+                yield event
+        except Exception as exception:
+            self._failed = True
+            _record_billable_partial_spend(self._logging_obj, self._collector)
+            mapped_exception: Final = self._map_exception(exception)
+            end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+            self._logging_obj.failure_handler(
+                mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
+            )
+            await self._logging_obj.async_failure_handler(
+                mapped_exception,
+                traceback.format_exc(),
+                self._start_time,
+                end_time,
+            )
+            raise mapped_exception from exception
+        finally:
+            await self._finalize()
+
+    async def _finalize(self) -> None:
+        if self._finalized or self._failed:
+            return
+        self._finalized = True
+        response: Final = self._collector.response()
+        end_time: Final = datetime.datetime.now()  # noqa: DTZ005  # callback timestamps use the legacy naive contract
+        self._logging_obj.handle_sync_success_callbacks_for_async_calls(
+            result=response,
+            start_time=self._start_time,
+            end_time=end_time,
+        )
+        with anyio.CancelScope(shield=True):
+            await self._logging_obj.async_success_handler(
+                result=response,
+                start_time=self._start_time,
+                end_time=end_time,
+            )
+
+    async def close(self) -> None:
+        try:
+            await super().close()
+        finally:
+            await self._finalize()
+
+    @property
+    def logging_obj(self) -> TranscriptionStreamLogging:
+        return self._logging_obj
+
+
+def wrap_transcription_stream(
+    stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
+    logging_obj: TranscriptionStreamLogging,
+    start_time: datetime.datetime,
+    map_exception: TranscriptionStreamExceptionMapper,
+) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
+    if isinstance(stream, AsyncStream):
+        return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
+    return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)

litellm/litellm_core_utils/audio_utils/utils.py

@@ -21,6 +21,16 @@ from litellm.types.files import (
 from litellm.types.utils import FileTypes
 
 
+def normalized_audio_duration_seconds(value: object) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    try:
+        seconds: Final = float(value)
+    except OverflowError:
+        return None
+    return seconds if math.isfinite(seconds) and seconds >= 0 else None
+
+
 @dataclass
 class ProcessedAudioFile:
     """

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -8,6 +8,7 @@ from typing import Final
 
 from pydantic import TypeAdapter, ValidationError
 
+from litellm._internal_context import internal_sub_call
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@@ -204,12 +205,13 @@ async def _execute_chat_completion_agentic_plan(
     )
 
     try:
-        response_followup = await litellm.acompletion(
-            model=full_model_name,
-            messages=patch.messages,
-            **optional_params_for_followup,
-            **kwargs_for_followup,
-        )
+        with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None):
+            response_followup = await litellm.acompletion(
+                model=full_model_name,
+                messages=patch.messages,
+                **optional_params_for_followup,
+                **kwargs_for_followup,
+            )
         if _post_hook_overridden(callback):
             try:
                 response_followup = await callback.async_post_agentic_loop_response_hook(

litellm/litellm_core_utils/core_helpers.py

@@ -4,7 +4,7 @@ import copy
 import logging
 import math
 import re
-from collections.abc import Collection, Iterable, Iterator, Mapping
+from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
 
@@ -97,13 +97,17 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
     return float(seconds / denominator)
 
 
-_DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
+_LAX_BOOL: Final = TypeAdapter(bool)
 
 
 def as_str_mapping(value: object) -> Mapping[str, object] | None:
     return value if isinstance(value, Mapping) else None
 
 
+def as_object_list(value: object) -> Sequence[object] | None:
+    return value if isinstance(value, list) else None
+
+
 def decoded_json_object(value: object) -> Mapping[str, object]:
     mapping: Final = as_str_mapping(value)
     if mapping is not None:
@@ -156,15 +160,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]:
     return iter_items(value) if value else ()
 
 
-def normalize_drop_params(value: object) -> bool | None:
+def lax_bool(value: object) -> bool | None:
     if value is None or isinstance(value, bool):
         return value
     try:
-        return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
+        return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
     except ValidationError:
         return None
 
 
+def normalize_drop_params(value: object) -> bool | None:
+    return lax_bool(value)
+
+
 def drops_unsupported_params(drop_params: object) -> bool:
     return litellm.drop_params is True or normalize_drop_params(drop_params) is True
 
@@ -311,9 +319,12 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "tool_use": "tool_calls",
     "refusal": "content_filter",
     "compaction": "length",
+    "pause_turn": "length",
     "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
+    "STOP_SEQUENCE": "stop",
+    "TOOL_CALL": "tool_calls",
     "ERROR_TOXIC": "content_filter",
     "ERROR": "stop",
     # HuggingFace / Together AI
@@ -1073,9 +1084,14 @@ def billed_usage_cost(response: object, custom_llm_provider: str | None) -> floa
 _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
 
 
+def reports_native_messages_cost(custom_llm_provider: object) -> bool:
+    """Whether a Messages upstream is one whose own reported cost litellm bills from instead of the price map."""
+    return isinstance(custom_llm_provider, str) and custom_llm_provider in _MESSAGES_REPORTED_COST_PROVIDERS
+
+
 def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
     """Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
-    if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
+    if not isinstance(custom_llm_provider, str) or not reports_native_messages_cost(custom_llm_provider):
         return None
     return provider_reported_usage_cost(usage, custom_llm_provider)
 

litellm/litellm_core_utils/decisions_pricing.py

@@ -12,6 +12,7 @@ from typing import Final
 
 DECISIONS_PRICING_SUFFIX: Final = "_decisions"
 DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
+QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
 
 
 def is_decisions_call_type(call_type: str) -> bool:
@@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
     }
 
 
+def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
+    if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
+        return None
+    rates: Final = {**model_info, **overrides}
+    request_price: Final = rates.get("input_cost_per_query")
+    if not isinstance(request_price, (int, float)):
+        return None
+    bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token"))
+    return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price)
+
+
 def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
     """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
     plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -107,6 +107,7 @@ class ExceptionCheckers:
             "exceeds the available context size",  # llama.cpp/Lemonade
             "exceeds the maximum number of tokens allowed",  # Gemini
             '"code": "context_length_exceeded"',
+            "prompt is too long",
         ]
         for substring in known_exception_substrings:
             if substring in _error_str_lowercase:

litellm/litellm_core_utils/get_litellm_params.py

@@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "github_copilot_host",
             "github_copilot_user_session",
             "fireworks_forward_user_id",
+            "thought_signature_in_tool_call_id",
             "responses_api_version",
             "allowed_openai_params",
             PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
 # Backward-compatible alias for existing imports/tests.
 _OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
 
+SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
+
 _CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
 _CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
 _MAX_SHOWN_INT_BITS: Final = 64