Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_langfuse_environment_dynamic_param

.github/CODEOWNERS

@@ -1,6 +1,9 @@
 /ui/ @yuneng-berri @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
+/ui/Dockerfile
+/ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
 /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri

.github/scripts/e2e_egress_sentinel.py

@@ -0,0 +1,198 @@
+"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
+
+`serve` pins each provider host (`--host`) to a local sink address in the hosts
+file and binds a counting listener on that address, so any connection the proxy
+or the record/replay edge opens to a real provider is redirected to the sink,
+recorded as one line in `--hits-file`, and never leaves the box. The record and
+replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
+pinned provider names), so in a clean replay the sink sees nothing; a single hit
+means a provider call escaped the bundle. `assert-empty` turns that hit file into
+the pass/fail check.
+
+Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
+editing the hosts file both need root); `--sink-address`, `--port`, and
+`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
+a high port under test.
+"""
+
+# ruff: noqa: T201  # CLI script: its stdout/stderr progress and results are the interface
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import Final
+
+_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
+_BLOCK_END: Final = "# END e2e-egress-sentinel"
+
+
+@dataclass(frozen=True, slots=True)
+class ServeConfig:
+    hosts: tuple[str, ...]
+    sink_address: str
+    ports: tuple[int, ...]
+    hits_file: Path
+    hosts_file: Path
+    ready_file: Path | None
+    pid_file: Path | None
+
+
+def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
+    lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
+    return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
+
+
+def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
+    original = hosts_file.read_bytes() if hosts_file.exists() else b""
+    hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
+    return original
+
+
+def _restore_pins(hosts_file: Path, original: bytes) -> None:
+    hosts_file.write_bytes(original)
+
+
+def _bind(sink_address: str, port: int) -> socket.socket:
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    listener.bind((sink_address, port))
+    listener.listen(128)
+    return listener
+
+
+@dataclass(frozen=True, slots=True)
+class _HitLog:
+    path: Path
+    _lock: threading.Lock
+
+    def record(self, *, port: int, peer: tuple[str, int]) -> None:
+        entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
+        with self._lock:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(entry + "\n")
+
+
+def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
+    while not stop.is_set():
+        try:
+            conn, peer = listener.accept()
+        except OSError:
+            return
+        hits.record(port=port, peer=(peer[0], peer[1]))
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def serve(config: ServeConfig) -> int:
+    config.hits_file.write_text("", encoding="utf-8")
+    original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
+    try:
+        listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
+    except OSError as exc:
+        _restore_pins(config.hosts_file, original_hosts)
+        print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
+        return 1
+
+    stop = threading.Event()
+    hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
+    threads = tuple(
+        threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
+        for listener, port in zip(listeners, config.ports)
+    )
+    for thread in threads:
+        thread.start()
+
+    def _handle(_signum: int, _frame: FrameType | None) -> None:
+        stop.set()
+        for listener in listeners:
+            try:
+                listener.close()
+            except OSError:
+                pass
+
+    signal.signal(signal.SIGTERM, _handle)
+    signal.signal(signal.SIGINT, _handle)
+
+    if config.pid_file is not None:
+        config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    if config.ready_file is not None:
+        config.ready_file.write_text("ready", encoding="utf-8")
+    print(
+        f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
+        f"on port(s) {', '.join(str(p) for p in config.ports)}",
+        flush=True,
+    )
+
+    stop.wait()
+    _restore_pins(config.hosts_file, original_hosts)
+    if config.ready_file is not None and config.ready_file.exists():
+        config.ready_file.unlink()
+    if config.pid_file is not None and config.pid_file.exists():
+        config.pid_file.unlink()
+    return 0
+
+
+def assert_empty(hits_file: Path) -> int:
+    if not hits_file.exists():
+        print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
+        return 0
+    hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not hits:
+        print("egress sentinel recorded no provider calls: zero egress")
+        return 0
+    print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
+    for line in hits:
+        print(f"  {line}", file=sys.stderr)
+    return 1
+
+
+def _serve_from_args(args: argparse.Namespace) -> int:
+    config = ServeConfig(
+        hosts=tuple(args.host),
+        sink_address=args.sink_address,
+        ports=tuple(args.port),
+        hits_file=Path(args.hits_file),
+        hosts_file=Path(args.hosts_file),
+        ready_file=Path(args.ready_file) if args.ready_file else None,
+        pid_file=Path(args.pid_file) if args.pid_file else None,
+    )
+    return serve(config)
+
+
+def main(argv: tuple[str, ...]) -> int:
+    parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
+    serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
+    serve_parser.add_argument("--sink-address", default="127.0.0.1")
+    serve_parser.add_argument("--port", action="append", type=int, default=None)
+    serve_parser.add_argument("--hits-file", required=True)
+    serve_parser.add_argument("--hosts-file", default="/etc/hosts")
+    serve_parser.add_argument("--ready-file", default=None)
+    serve_parser.add_argument("--pid-file", default=None)
+
+    assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
+    assert_parser.add_argument("--hits-file", required=True)
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        if args.port is None:
+            args.port = [443]
+        return _serve_from_args(args)
+    return assert_empty(Path(args.hits_file))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(tuple(sys.argv[1:])))

.github/scripts/e2e_fetch_fixture_bundle.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
+ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
+BASE_BRANCH="${3:?base branch required}"
+DEST_DIR="${4:?destination bundle dir required}"
+
+: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
+
+SELECTED="$(
+  gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
+    --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
+    | jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
+)"
+
+if [[ -z "${SELECTED}" ]]; then
+  echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
+  exit 1
+fi
+
+RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
+RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
+ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
+GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
+CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
+
+echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
+
+gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
+
+TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
+if [[ -z "${TARBALL}" ]]; then
+  echo "downloaded artifact contained no tarball" >&2
+  exit 1
+fi
+SIDECAR="${TARBALL}.sha256"
+if [[ ! -f "${SIDECAR}" ]]; then
+  echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
+  exit 1
+fi
+
+echo "verifying bundle against its recorded sha256 digest"
+( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
+
+mkdir -p "${DEST_DIR}"
+tar xzf "${TARBALL}" -C "${DEST_DIR}"
+
+echo "extracted bundle into ${DEST_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

.github/scripts/e2e_pack_fixture_bundle.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <bundle-dir> <out-tarball>" >&2
+  exit 2
+fi
+
+BUNDLE_DIR="$1"
+OUT_TARBALL="$2"
+
+MANIFEST="${BUNDLE_DIR}/manifest.json"
+if [[ ! -f "${MANIFEST}" ]]; then
+  echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
+  exit 1
+fi
+
+echo "packing fixture bundle from ${BUNDLE_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
+
+TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
+if [[ "${TEST_DIRS}" -eq 0 ]]; then
+  echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
+  exit 1
+fi
+echo "  ${TEST_DIRS} recorded test director(ies)"
+
+mkdir -p "$(dirname "${OUT_TARBALL}")"
+tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
+
+OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
+OUT_BASE="$(basename "${OUT_TARBALL}")"
+( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
+
+echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
+cat "${OUT_DIR}/${OUT_BASE}.sha256"

.github/workflows/e2e_record_replay.yml

@@ -0,0 +1,237 @@
+name: "E2E Record and Replay"
+
+on:
+  schedule:
+    - cron: "0 8 * * 6"
+    - cron: "0 8 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
+        type: choice
+        options:
+          - record
+          - replay
+        default: record
+
+permissions:
+  contents: read
+
+jobs:
+  record:
+    name: "Record the e2e suite against real providers"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 6' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Record the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: record
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Pack the fixture bundle
+        run: |
+          .github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
+
+      - name: Publish the fixture bundle
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-fixtures-bundle
+          path: |
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log
+
+  replay:
+    name: "Replay the e2e suite from the pinned bundle with zero egress"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 1-5' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: read
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      GH_TOKEN: ${{ github.token }}
+      OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
+      ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Fetch the pinned fixture bundle by digest
+        env:
+          BASE_BRANCH: ${{ github.ref_name }}
+        run: |
+          .github/scripts/e2e_fetch_fixture_bundle.sh \
+            "${GITHUB_REPOSITORY}" \
+            e2e-fixtures-bundle \
+            "${BASE_BRANCH}" \
+            tests/e2e/.fixtures
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Start the egress sentinel
+        run: |
+          # shellcheck disable=SC2024  # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
+          sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
+            --host api.openai.com \
+            --host api.anthropic.com \
+            --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
+            --ready-file "${RUNNER_TEMP}/egress-ready" \
+            --pid-file "${RUNNER_TEMP}/egress.pid" \
+            > "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
+              cat "${RUNNER_TEMP}/egress-sentinel.log"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "egress sentinel never became ready"
+          cat "${RUNNER_TEMP}/egress-sentinel.log"
+          exit 1
+
+      - name: Replay the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: replay
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Stop the egress sentinel and assert zero provider egress
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
+            sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
+            sleep 2
+          fi
+          python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/image-scan.yml

@@ -23,6 +23,8 @@ on:
       - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
+      - ui/Dockerfile
+      - ui/nginx.conf
       - .github/workflows/image-scan.yml
   schedule:
     - cron: "41 6 * * *"
@@ -185,6 +187,35 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
 
+  ui-image:
+    name: ui-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build UI image
+        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
+        env:
+          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
+
   backend-image:
     name: backend-image
     runs-on: ubuntu-latest

.github/workflows/test-code-quality.yml

@@ -131,6 +131,9 @@ jobs:
       - name: check_e2e_no_raw_requests
         run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
 
+      - name: check_migrations_no_data_rewrites
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-unit.yml

@@ -211,7 +211,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-extras
             artifact-name: proxy-extras
@@ -219,7 +219,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: enterprise-package
             artifact-name: enterprise-package
@@ -227,7 +227,7 @@ jobs:
             workers: 4
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types

CLAUDE.md

@@ -79,6 +79,8 @@ Do not put names of customers or customer company names in code, PR descriptions
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
+
 Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
 
 - Composition over inheritance

basedpyright-code-budget.json

@@ -1,6 +1,6 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 19949
   },
   "reportArgumentType": {
     "limit": 2566
@@ -54,7 +54,7 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5661
   },
   "reportMissingTypeArgument": {
     "limit": 15555
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1822
+    "limit": 1810
   },
   "reportRedeclaration": {
     "limit": 8
@@ -105,10 +105,10 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39011
+    "limit": 39009
   },
   "reportUnknownParameterType": {
-    "limit": 19885
+    "limit": 19883
   },
   "reportUnknownVariableType": {
     "limit": 30569

helm/litellm/values.yaml

@@ -428,9 +428,11 @@ ui:
     maxUnavailable: ""
   podAnnotations: {}
   # Same shape as the gateway blocks of the same name. The nginx runtime
-  # writes its pid, cache, and proxy temp files under the image's root
-  # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
-  # emptyDir volumes mounted over those paths.
+  # writes its pid, cache, and proxy temp files under /tmp, so it boots as
+  # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
+  # true` here needs an emptyDir volume mounted over /tmp. Images before
+  # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
+  # run as a non-root uid at all.
   podLabels: {}
   podSecurityContext: {}
   securityContext: {}

litellm/__init__.py

@@ -199,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
 )
 log_raw_request_response: bool = False
+log_client_error_tracebacks: bool = False
 request_correlation_in_logs: bool = False
 redact_messages_in_exceptions: Optional[bool] = False
 redact_user_api_key_info: Optional[bool] = False
@@ -1628,6 +1629,9 @@ if TYPE_CHECKING:
         AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -1801,6 +1805,9 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,

litellm/_lazy_imports_registry.py

@@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
+    "TogetherAIChatConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
     "GoogleAIStudioGeminiConfig",
@@ -242,6 +243,7 @@ LLM_CONFIG_NAMES: Final = (
     "OpenRouterResponsesAPIConfig",
     "BedrockMantleResponsesAPIConfig",
     "GoogleAIStudioInteractionsConfig",
+    "VertexAIInteractionsConfig",
     "OpenAIOSeriesConfig",
     "AnthropicSkillsConfig",
     "BaseSkillsAPIConfig",
@@ -740,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "AmazonMantleMessagesConfig",
     ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
+    "TogetherAIChatConfig": (
+        ".llms.together_ai.chat.transformation",
+        "TogetherAIChatConfig",
+    ),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (
         ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",
@@ -977,6 +983,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.gemini.interactions.transformation",
         "GoogleAIStudioInteractionsConfig",
     ),
+    "VertexAIInteractionsConfig": (
+        ".llms.vertex_ai.interactions.transformation",
+        "VertexAIInteractionsConfig",
+    ),
     "OpenAIOSeriesConfig": (
         ".llms.openai.chat.o_series_transformation",
         "OpenAIOSeriesConfig",

litellm/a2a_protocol/card_resolver.py

@@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
 Extends the A2A SDK's card resolver to support multiple well-known paths.
 """
 
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
@@ -48,6 +49,43 @@ def is_localhost_or_internal_url(url: str | None) -> bool:
     return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
 
 
+_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
+    {
+        "jsonrpc": "JSONRPC",
+        "http+json": "HTTP+JSON",
+        "grpc": "GRPC",
+    }
+)
+
+_LEGACY_PROTOCOL_VERSION: Final = "0.3"
+
+
+def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
+    """
+    Canonicalize the supported interfaces of spec-adjacent agent cards.
+
+    Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
+    bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
+    case-sensitively against its uppercase TransportProtocol constants and fails
+    with "no compatible transports found." for spec-adjacent casings.
+
+    The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
+    payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
+    proto parsing rejects. A mis-cased binding fingerprints such a server, so its
+    declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
+    its v0.3 compat transport, which speaks that dialect.
+    """
+    normalized: Final = type(agent_card)()
+    normalized.CopyFrom(agent_card)
+    for interface in normalized.supported_interfaces:
+        canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower())
+        if canonical is None or canonical == interface.protocol_binding:
+            continue
+        interface.protocol_binding = canonical
+        interface.protocol_version = _LEGACY_PROTOCOL_VERSION
+    return normalized
+
+
 def get_agent_card_url(agent_card: "AgentCard") -> str | None:
     """Return the agent endpoint URL from the resolved SDK card."""
     url: Final = getattr(agent_card, "url", None)

litellm/a2a_protocol/main.py

@@ -73,6 +73,7 @@ except ImportError:
 from litellm.a2a_protocol.card_resolver import (
     LiteLLMA2ACardResolver,
     get_agent_card_url,
+    normalize_agent_card_interfaces,
 )
 from litellm.a2a_protocol.exception_mapping_utils import (
     handle_a2a_localhost_retry,
@@ -782,13 +783,17 @@ async def create_a2a_client(
     if extra_headers:
         verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
 
+    resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+    agent_card: Final = normalize_agent_card_interfaces(
+        await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None)
+    )
+
     a2a_client: Final = await create_client(  # pyright: ignore[reportOptionalCall]
-        base_url,
+        agent_card,
         client_config=ClientConfig(  # pyright: ignore[reportOptionalCall]
             httpx_client=httpx_client,
             streaming=streaming,
         ),
-        resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
     )
     # Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
     # the configured httpx client and this agent's headers without excavating
@@ -799,9 +804,7 @@ async def create_a2a_client(
         if extra_headers
         else None
     )
-    agent_card: Final = getattr(a2a_client, "_card", None)
-    if agent_card is not None:
-        a2a_client._litellm_agent_card = agent_card
+    a2a_client._litellm_agent_card = agent_card
 
     verbose_logger.info("A2A client created for %s", base_url)
 

litellm/caching/redis_cluster_node_isolation.py

@@ -18,6 +18,14 @@ already does when one of its pooled connections errors), leaving every other nod
 connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
+
+redis-py 8.x fixed this upstream with gentler machinery than this override's
+``node.disconnect()`` (which also kills connections other coroutines are mid-operation
+on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
+killed connection): it marks in-use connections for reconnect only after their current
+operation completes, disconnects only the idle pooled ones, and defers reinitialization
+to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
+recovery API, the factory returns the base ``RedisCluster`` unmodified.
 """
 
 import asyncio
@@ -72,8 +80,16 @@ class _ClusterAttrs(Protocol):
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
 
 
-def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
-    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+def get_litellm_async_redis_cluster_class(
+    cluster_node_class: type | None = None,
+) -> type["_AsyncRedisClusterType"]:
+    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
+    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch
+    tears down the whole cluster client.
+
+    ``cluster_node_class`` exists for dependency injection in tests; production callers
+    leave it unset and the installed ``ClusterNode`` is used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -81,7 +97,10 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     """
     import redis
     from redis.asyncio.cluster import (
-        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+        ClusterNode as _AsyncClusterNode,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # same stale-stub gap as the import above
     )
     from redis.cluster import get_node_name
     from redis.commands import READ_COMMANDS
@@ -98,6 +117,15 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     from redis.exceptions import ConnectionError as _RedisConnectionError
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
+    node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    if hasattr(node_class, "update_active_connections_for_reconnect"):
+        verbose_logger.debug(
+            "redis-py %s recovers a node-level connection error per-connection upstream; "
+            "using the base RedisCluster without litellm's node-isolation override.",
+            redis.__version__,
+        )
+        return _BaseAsyncRedisCluster
+
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(
             "redis-py %s is not in the set this cluster-teardown-storm fix was verified "

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -21,6 +21,9 @@ from pydantic import BaseModel
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    responses_reasoning_item_from_thinking_blocks,
+)
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.bridges.completion_transformation import (
     CompletionTransformationBridge,
@@ -32,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
 )
 from litellm.responses.utils import normalize_responses_api_stream_options
 from litellm.types.llms.openai import (
+    REASONING_EFFORT,
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
     ChatCompletionToolCallChunk,
@@ -85,6 +89,22 @@ def _get_reasoning_items(
     return []
 
 
+def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]:  # mutable-ok: API message payload
+    """Reasoning input items for an assistant message.
+
+    Stored reasoning items win because they carry an id the Responses API minted; thinking
+    blocks are the fallback for turns that arrived over another API surface.
+    """
+    items: Final = _get_reasoning_items(msg)
+    stored: Final = [_reasoning_item_to_response_input(item) for item in items]  # mutable-ok: API message payload
+    if stored:
+        return stored
+    raw_blocks: Final = msg.get("thinking_blocks") or ()
+    blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks)  # cast-ok: untyped client json
+    from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
+    return [] if from_thinking is None else [dict(from_thinking)]  # mutable-ok: API message payload
+
+
 def _build_reasoning_item(
     item_id: str,
     encrypted_content: str | None,
@@ -372,8 +392,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         )
                     )
             elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
-                for r_item in _get_reasoning_items(msg):
-                    input_items.append(_reasoning_item_to_response_input(r_item))
+                input_items.extend(_reasoning_input_items(msg))
+                if content:
+                    input_items.append(
+                        {  # mutable-ok: API message payload
+                            "type": "message",
+                            "role": "assistant",
+                            "content": self._convert_content_to_responses_format(content, "assistant"),
+                        }
+                    )
                 for tool_call in tool_calls:
                     function = tool_call.get("function")
                     custom = tool_call.get("custom")
@@ -400,15 +427,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         raise ValueError(f"tool call not supported: {tool_call}")
             elif content is not None:
                 if role == "assistant":
-                    for r_item in _get_reasoning_items(msg):
-                        input_items.append(_reasoning_item_to_response_input(r_item))
+                    input_items.extend(_reasoning_input_items(msg))
                 input_items.append(
-                    {
+                    {  # mutable-ok: API message payload
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(content, cast(str, role)),
                     }
                 )
+            elif role == "assistant":
+                input_items.extend(_reasoning_input_items(msg))
 
         return input_items, instructions
 
@@ -1086,22 +1114,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
         )
 
-        # If string is passed, map with optional summary based on flag/env var
-        if reasoning_effort == "none":
-            return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
-        elif reasoning_effort == "high":
-            return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
-        elif reasoning_effort == "xhigh":
-            return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
-        elif reasoning_effort == "medium":
+        if reasoning_effort in get_args(REASONING_EFFORT):
             return (
-                Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
-            )
-        elif reasoning_effort == "low":
-            return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
-        elif reasoning_effort == "minimal":
-            return (
-                Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
+                Reasoning(effort=reasoning_effort, summary="detailed")
+                if auto_summary_enabled
+                else Reasoning(effort=reasoning_effort)
             )
         return None
 

litellm/constants.py

@@ -48,6 +48,7 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Data URIs exceeding this are replaced with a size placeholder.
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
+REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
 
 MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
 
@@ -146,6 +147,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-adaptive-router-model",
     "x-litellm-applied-guardrails",
     "x-litellm-guardrail-scan-id",
+    "x-litellm-cache-key",
 ]
 
 # Gemini model-specific minimal thinking budget constants
@@ -749,6 +751,7 @@ openai_compatible_endpoints: Final[list] = [
     "api.groq.com/openai/v1",
     "https://integrate.api.nvidia.com/v1",
     "api.deepseek.com/v1",
+    "api.together.ai/v1",
     "api.together.xyz/v1",
     "app.empower.dev/api/v1",
     "https://api.friendli.ai/serverless/v1",
@@ -1563,6 +1566,19 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEA
 # installations with large numbers of stale managed objects).
 _batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
 PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true"
+BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5")
+)
+BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60")
+)
+BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600")
+)
+_background_interaction_cost_polling_env: Final = os.getenv(
+    "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
+).lower()
+BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
 PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
 PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10))  # in seconds, increased from 10
 PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

litellm/cost_calculator.py

@@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
 from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    InteractionsUsageObjectTransformation,
     TranscriptionUsageObjectTransformation,
 )
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
@@ -150,6 +151,7 @@ _VIDEO_CALL_TYPES: Final = frozenset(
     }
 )
 
+
 _SPEECH_CALL_TYPES: Final = frozenset(
     {
         CallTypes.speech.value,
@@ -912,6 +914,8 @@ def _get_usage_object(
                 usage_obj,
             )
         )
+    elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj):
+        return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj)
     elif isinstance(usage_obj, dict):
         return Usage(**usage_obj)
     elif isinstance(usage_obj, BaseModel):
@@ -1288,6 +1292,10 @@ def completion_cost(
                         )
                         if tr_usage is not None:
                             _usage = tr_usage.model_dump()
+                    elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage):
+                        _usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(
+                            _usage
+                        ).model_dump()
                     else:
                         _usage = _usage
 
@@ -1372,23 +1380,36 @@ def completion_cost(
                     if custom_pricing and litellm_logging_obj is not None:
                         _litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
                         if _litellm_params is not None:
-                            _metadata = _litellm_params.get("metadata", {}) or {}
-                            _video_model_info = _metadata.get("model_info", None)
+                            _video_model_info = next(
+                                (
+                                    model_info
+                                    for _metadata_key in ("metadata", "litellm_metadata")
+                                    if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
+                                    is not None
+                                ),
+                                None,
+                            )
 
                     usage_obj = getattr(completion_response, "usage", None)
                     duration_seconds: float | None = None
                     video_resolution: str | None = None
+                    provider_reported_cost: float | None = None
                     if completion_response is not None and usage_obj:
                         # Handle both dict and Pydantic Usage object
                         if isinstance(usage_obj, dict):
                             duration_seconds = usage_obj.get("duration_seconds", None)
                             _vr = usage_obj.get("video_resolution", None)
+                            provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
                         else:
                             duration_seconds = getattr(usage_obj, "duration_seconds", None)
                             _vr = getattr(usage_obj, "video_resolution", None)
+                            provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
                         if _vr is not None:
                             video_resolution = str(_vr).strip().lower()
 
+                        if _video_model_info is None and provider_reported_cost is not None:
+                            return float(provider_reported_cost)
+
                         if duration_seconds is not None:
                             # Calculate cost based on video duration using video-specific cost calculation
                             from litellm.llms.openai.cost_calculation import (

litellm/images/main.py

@@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
 from litellm.exceptions import LiteLLMUnknownProvider
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values
 from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -422,24 +423,32 @@ def image_generation(
                 aimg_generation=aimg_generation,
             )
         elif custom_llm_provider == "azure_ai":
-            from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+            from litellm.llms.azure_ai.common_utils import (
+                AzureFoundryModelInfo,
+                get_azure_ai_auth_headers,
+            )
 
             api_base = AzureFoundryModelInfo.get_api_base(api_base)
             api_key = AzureFoundryModelInfo.get_api_key(api_key)
             if extra_headers is not None:
                 optional_params["extra_headers"] = extra_headers
 
-            default_headers = {
+            caller_header_names = frozenset(name.lower() for name in headers)
+            caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names
+            auth_headers = (
+                headers
+                if caller_set_auth
+                else get_azure_ai_auth_headers(
+                    api_key=api_key,
+                    litellm_params=litellm_params_dict,
+                    api_key_header="api-key",
+                )
+            )
+            request_headers: Final = {
                 "Content-Type": "application/json",
+                **auth_headers,
+                **headers,
             }
-            # Only add api-key header if api_key is not None
-            # Azure AD authentication will use Authorization header instead
-            if api_key is not None:
-                default_headers["api-key"] = api_key
-
-            for k, v in default_headers.items():
-                if k not in headers:
-                    headers[k] = v
 
             model_response = azure_chat_completions.image_generation(
                 model=model,
@@ -455,7 +464,7 @@ def image_generation(
                 api_version=api_version,
                 aimg_generation=aimg_generation,
                 client=client,
-                headers=headers,
+                headers=request_headers,
                 litellm_params=litellm_params_dict,
             )
         elif (
@@ -846,6 +855,18 @@ def image_edit(
             additional_drop_params=kwargs.get("additional_drop_params"),
         )
 
+        if (
+            custom_llm_provider == "openai"
+            or custom_llm_provider == "azure"
+            or custom_llm_provider in litellm.openai_compatible_providers
+        ):
+            image_edit_request_params.update(
+                flatten_form_field_values(
+                    non_default_params,
+                    extra_body if isinstance(extra_body, dict) else None,
+                )
+            )
+
         # Pre Call logging
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
@@ -995,6 +1016,9 @@ async def aimage_edit(
             response_format=response_format,
             size=size,
             user=user,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
             timeout=timeout,
             custom_llm_provider=custom_llm_provider,
             **kwargs,

litellm/integrations/langsmith.py

@@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
         return outputs
 
     def _ensure_required_ids(self, data: dict, run_id: str | None):
+        resolved_id: Final = run_id or str(uuid.uuid4())
         if "id" not in data or data["id"] is None:
-            run_id = str(uuid.uuid4())
-            data["id"] = run_id
+            data["id"] = resolved_id
 
-        if "trace_id" not in data or data["trace_id"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["trace_id"] = run_id
+        # LangSmith rejects the whole ingest batch unless a root run's trace_id
+        # equals the run id embedded in the first segment of dotted_order
+        posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
+            "dotted_order" not in data or data["dotted_order"] is None
+        )
+        if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
+            data["trace_id"] = resolved_id
 
         if "dotted_order" not in data or data["dotted_order"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["dotted_order"] = self.make_dot_order(run_id=run_id)
+            data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
 
     def _prepare_log_data(
         self,
@@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
             metadata = _litellm_params.get("metadata", {}) or {}
 
             fields: Final = self._extract_metadata_fields(metadata, credentials)
+            # the proxy header fan-out mirrors one value into both keys, and LangSmith
+            # rejects the whole ingest batch when run-body session_id is not an
+            # existing tracer-session uuid
+            if fields["session_id"] == fields["trace_id"]:
+                fields["session_id"] = None
             verbose_logger.debug(
                 "Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
             )

litellm/integrations/s3_v2.py

@@ -11,6 +11,7 @@ import time
 from collections.abc import Mapping
 from datetime import datetime
 from typing import Final, cast
+from urllib.parse import quote
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
         )
 
+    def _build_object_url(self, s3_object_key: str) -> str:
+        """
+        Build the exact URL that is both signed and sent, with the key percent-encoded once.
+
+        S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
+        characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
+        the two signatures disagree (403 SignatureDoesNotMatch).
+        """
+        encoded_key: Final = quote(s3_object_key, safe="/")
+        if self.s3_endpoint_url and self.s3_bucket_name:
+            if self.s3_use_virtual_hosted_style:
+                endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
+                protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
+                return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
+            return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
+        return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
+
     def _sse_headers(self) -> Mapping[str, str]:
         candidates: Final = {
             "x-amz-server-side-encryption": self.s3_server_side_encryption,
@@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
             verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
             from botocore.credentials import Credentials
@@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 aws_region_name=self.s3_region_name,
             )
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             httpx_client: Final = _get_httpx_client(
                 params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
             )
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         try:
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
 
             verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
+            url: Final = self._build_object_url(s3_object_key)
 
             # Prepare the request for GET operation
             # For GET requests, we need x-amz-content-sha256 with hash of empty string
@@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             headers: Final = {
                 "x-amz-content-sha256": empty_string_hash,
             }
-            req: Final = requests.Request("GET", url, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
             S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            request_url: Final = prepped.url or url
-            response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
+            response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
 
             if response.status_code != 200:
                 verbose_logger.exception("S3 object not found, saw response=", response.text)

litellm/interactions/background_cost_polling.py

@@ -0,0 +1,313 @@
+"""
+Cost tracking for background interactions.
+
+A create request with ``background=true`` returns ``in_progress`` with no
+usage block, and GET polls are deliberately never billed (billing them would
+double-charge every poll; the GET response also does not echo ``background``,
+so a poll cannot be told apart from a re-fetch of an already-billed
+interaction). The create call is therefore the only place that can own
+billing: it schedules a poll task that fetches the interaction until it
+reaches a terminal status and logs the final usage as a single success event
+attributed to the original request.
+
+``requires_action`` is terminal for the interaction it names. The API has no
+operation that resumes one: a caller answers a tool request by creating a new
+interaction whose ``previous_interaction_id`` points at it, and that new
+interaction bills itself. The paused interaction keeps the tokens it already
+spent producing the tool request, so it is billed and settled where it stops
+rather than polled until the timeout, which would both lose that usage and
+hold its budget reservation open for the whole timeout window.
+
+Deleting an interaction makes every subsequent poll fail, which would let a
+caller retrieve the completed output themselves and then delete it before the
+poll task settles, leaving the work unbilled and the budget reservation
+refunded at the poll timeout. ``adelete`` therefore settles any pending poll
+for the interaction before dispatching the delete: it fetches the current
+state with the create's credentials, bills it if it is terminal with usage,
+and releases the reservation otherwise. A settlement gate on the create's
+logging object makes the poll task and the delete path mutually exclusive, so
+the interaction is billed exactly once no matter who settles first.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+from litellm._logging import verbose_logger
+from litellm.constants import (
+    BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
+)
+from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
+from litellm.types.interactions import InteractionsAPIResponse
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_TERMINAL_STATUSES: Final = frozenset(
+    {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
+)
+
+_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
+
+_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
+
+
+@dataclass(frozen=True, slots=True)
+class BackgroundInteractionPollContext:
+    interaction_id: str
+    custom_llm_provider: str
+    logging_obj: "LiteLLMLoggingObj"
+    api_key: str | None = None
+    api_base: str | None = None
+    initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
+    max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
+    timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
+
+
+FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]]
+
+
+async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
+    from litellm.interactions import aget
+
+    return await aget(
+        interaction_id=context.interaction_id,
+        custom_llm_provider=context.custom_llm_provider,
+        api_key=context.api_key,
+        api_base=context.api_base,
+        **{
+            "no-log": True
+        },  # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
+    )
+
+
+def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
+    elapsed = 0.0
+    interval = initial
+    while interval > 0 and elapsed + interval <= timeout:
+        yield interval
+        elapsed += interval
+        interval = min(interval * 2, maximum)
+
+
+_SETTLED_KEY = "background_interaction_settled"
+
+
+def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool:
+    return logging_obj.model_call_details.get(_SETTLED_KEY) is True
+
+
+def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool:
+    """
+    Exactly-once gate between the poll task and the delete-time settlement:
+    both run on the same event loop and neither awaits between reading and
+    setting the flag, so whichever claims first owns billing or release.
+    """
+    if _is_settled(logging_obj):
+        return False
+    logging_obj.model_call_details[_SETTLED_KEY] = True  # rebind-ok: both settlers must see the same settlement flag
+    return True
+
+
+async def poll_and_log_background_interaction_cost(
+    context: BackgroundInteractionPollContext,
+    fetch_interaction: FetchInteraction = _fetch_interaction,
+) -> None:
+    last_seen_status: str | None = None
+    for interval in _poll_intervals(
+        initial=context.initial_interval_seconds,
+        maximum=context.max_interval_seconds,
+        timeout=context.timeout_seconds,
+    ):
+        await asyncio.sleep(interval)
+        if _is_settled(context.logging_obj):
+            return
+        try:
+            response = await fetch_interaction(context)
+        except Exception as e:  # noqa: BLE001  # any fetch error must not kill the billing poll loop
+            verbose_logger.debug(
+                "Background interaction cost poll for %s failed, will retry: %s",
+                context.interaction_id,
+                e,
+            )
+            continue
+        last_seen_status = response.status
+        if response.status not in _TERMINAL_STATUSES:
+            continue
+        if not _claim_settlement(context.logging_obj):
+            return
+        if response.usage is not None:
+            await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+        else:
+            await _release_open_budget_reservation(logging_obj=context.logging_obj)
+        return
+    if not _claim_settlement(context.logging_obj):
+        return
+    if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES:
+        verbose_logger.error(
+            "Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
+            "the pollable nor the terminal set, so this proxy never learned how to settle it and its usage "
+            "will not be tracked",
+            context.interaction_id,
+            context.timeout_seconds,
+            last_seen_status,
+        )
+    else:
+        verbose_logger.warning(
+            "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
+            context.interaction_id,
+            context.timeout_seconds,
+        )
+    await _release_open_budget_reservation(logging_obj=context.logging_obj)
+
+
+async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
+    """
+    The proxy keeps the pre-call budget reservation open for an in-progress
+    background interaction so concurrent creates cannot stack past the budget.
+    The completion success event reconciles it to the actual cost; when the
+    interaction terminates without billable usage (or polling gives up, or it
+    is deleted before settling), no such event fires, so whoever claims the
+    settlement must release the reservation here or the spend counters stay
+    pinned at the estimated cost.
+    """
+    metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
+    budget_reservation = metadata.get("user_api_key_budget_reservation")
+    if not isinstance(budget_reservation, dict):
+        return
+
+    from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation
+
+    try:
+        await release_budget_reservation(budget_reservation=budget_reservation)
+    except Exception:  # noqa: BLE001  # a failed release must not crash the poll task; counters expire via TTL
+        verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction")
+
+
+async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None:
+    """
+    Claiming the settlement makes the claimer solely responsible for the
+    reservation, and no one retries a claim that is already set. A billing
+    failure here must therefore release the reservation on its way out, or it
+    stays pinned at the estimated cost until the whole poll times out.
+    """
+    try:
+        await logging_obj.async_log_background_interaction_completion(result=response)
+    except Exception:
+        await _release_open_budget_reservation(logging_obj=logging_obj)
+        raise
+
+
+def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool:
+    """
+    The single gate deciding whether a create's response gets a poll task.
+    The proxy's success callback defers releasing the budget reservation for
+    exactly these responses, on the promise that a poll task will settle them,
+    so a response one site accepts and the other refuses strands its
+    reservation on the spend counters with nothing left to reconcile it.
+
+    ``queued`` belongs here alongside ``in_progress``. It is the API's
+    not-started-yet state, so it reaches a terminal status the same way and
+    needs polling for the same reason: nothing else in the proxy ever bills a
+    create that came back without usage, so a status missing from both this
+    set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody.
+    """
+    return response.status in _POLLABLE_STATUSES and bool(response.id)
+
+
+def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool:
+    """
+    Whether a response arriving with no usage block is a normal outcome rather
+    than lost billing data. An interaction that is still running, or that
+    stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``,
+    has nothing to charge for and should not raise a cost-tracking alarm.
+
+    ``completed`` and ``requires_action`` both mean the model produced output,
+    so a usage block is always expected with them. If one arrives without it
+    the charge for real work has been lost, which is precisely what the
+    proxy's cost-tracking alert exists to surface.
+    """
+    return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT
+
+
+@dataclass(frozen=True, slots=True)
+class _ActiveBackgroundPoll:
+    task: "asyncio.Task[None]"
+    context: BackgroundInteractionPollContext
+
+
+_ACTIVE_POLLS: dict[str, _ActiveBackgroundPoll] = {}  # mutable-ok: asyncio needs strong refs to running poll tasks
+
+
+def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None:
+    entry = _ACTIVE_POLLS.get(interaction_id)
+    if entry is not None and entry.task is task:
+        del _ACTIVE_POLLS[interaction_id]
+
+
+def maybe_schedule_background_interaction_cost_polling(
+    response: object,
+    create_kwargs: Mapping[str, object],
+    custom_llm_provider: str,
+) -> "asyncio.Task[None] | None":
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
+        return None
+    if not isinstance(response, InteractionsAPIResponse):
+        return None
+    if not is_pollable_background_interaction(response):
+        return None
+    logging_obj = create_kwargs.get("litellm_logging_obj")
+    if not isinstance(logging_obj, Logging):
+        return None
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return None
+    api_key = create_kwargs.get("api_key")
+    api_base = create_kwargs.get("api_base")
+    context = BackgroundInteractionPollContext(
+        interaction_id=response.id,
+        custom_llm_provider=custom_llm_provider,
+        logging_obj=logging_obj,
+        api_key=api_key if isinstance(api_key, str) else None,
+        api_base=api_base if isinstance(api_base, str) else None,
+    )
+    task = asyncio.create_task(poll_and_log_background_interaction_cost(context))
+    _ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context)
+    task.add_done_callback(
+        lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished)
+    )
+    return task
+
+
+async def maybe_settle_background_interaction_before_delete(
+    interaction_id: str,
+    fetch_interaction: FetchInteraction = _fetch_interaction,
+) -> None:
+    entry = _ACTIVE_POLLS.get(interaction_id)
+    if entry is None:
+        return
+    context = entry.context
+    try:
+        response = await fetch_interaction(context)
+    except Exception as e:  # noqa: BLE001  # unfetchable pre-delete state settles by releasing the reservation
+        verbose_logger.debug(
+            "Could not fetch background interaction %s before delete, releasing its reservation: %s",
+            interaction_id,
+            e,
+        )
+        if _claim_settlement(context.logging_obj):
+            await _release_open_budget_reservation(logging_obj=context.logging_obj)
+        return
+    if not _claim_settlement(context.logging_obj):
+        return
+    if response.status in _TERMINAL_STATUSES and response.usage is not None:
+        await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+        return
+    await _release_open_budget_reservation(logging_obj=context.logging_obj)

litellm/interactions/main.py

@@ -40,6 +40,10 @@ from typing import Any, Final
 import httpx
 
 import litellm
+from litellm.interactions.background_cost_polling import (
+    maybe_schedule_background_interaction_cost_polling,
+    maybe_settle_background_interaction_before_delete,
+)
 from litellm.interactions.http_handler import interactions_http_handler
 from litellm.interactions.utils import (
     InteractionsAPIRequestUtils,
@@ -171,6 +175,12 @@ async def acreate(
         else:
             response = init_response
 
+        maybe_schedule_background_interaction_cost_polling(
+            response=response,
+            create_kwargs=kwargs,
+            custom_llm_provider=custom_llm_provider,
+        )
+
         return response
     except Exception as e:
         raise litellm.exception_type(
@@ -462,6 +472,8 @@ async def adelete(
         loop: Final = asyncio.get_event_loop()
         kwargs["adelete_interaction"] = True
 
+        await maybe_settle_background_interaction_before_delete(interaction_id=interaction_id)
+
         func: Final = partial(
             delete,
             interaction_id=interaction_id,

litellm/interactions/utils.py

@@ -47,6 +47,13 @@ def get_provider_interactions_api_config(
 
         return GoogleAIStudioInteractionsConfig()
 
+    if provider in (LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value):
+        from litellm.llms.vertex_ai.interactions.transformation import (
+            VertexAIInteractionsConfig,
+        )
+
+        return VertexAIInteractionsConfig()
+
     return None
 
 

litellm/litellm_core_utils/core_helpers.py

@@ -58,6 +58,26 @@ def safe_divide(
     return numerator / denominator
 
 
+def is_expected_client_error(exception: BaseException | None) -> bool:
+    """
+    True when the exception maps to an HTTP 4xx status.
+
+    ProxyException stores the status on .code (as a str), HTTPException and
+    litellm exceptions on .status_code.
+    """
+    if exception is None:
+        return False
+    code: Final[object] = getattr(exception, "code", None)
+    status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
+    if status_code is None or isinstance(status_code, bool):
+        return False
+    try:
+        status: Final = int(str(status_code))
+    except ValueError:
+        return False
+    return 400 <= status < 500
+
+
 def coerce_token_limit(value: object) -> int | None:
     """
     Coerce a max_input_tokens / max_output_tokens value to an int, treating a

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2301,6 +2301,7 @@ def exception_type(
                 or custom_llm_provider == "custom_openai"
                 or custom_llm_provider in litellm.openai_compatible_providers
                 or custom_llm_provider == "mistral"
+                or custom_llm_provider == "runwayml"
             ):
                 _map_openai_exception(
                     model=model,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -272,6 +272,14 @@ def get_llm_provider(
                     elif endpoint == "api.deepseek.com/v1":
                         custom_llm_provider = "deepseek"
                         dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
+                    elif endpoint == "api.together.ai/v1" or endpoint == "api.together.xyz/v1":
+                        custom_llm_provider = "together_ai"
+                        dynamic_api_key = api_key or (
+                            get_secret_str("TOGETHER_API_KEY")
+                            or get_secret_str("TOGETHER_AI_API_KEY")
+                            or get_secret_str("TOGETHERAI_API_KEY")
+                            or get_secret_str("TOGETHER_AI_TOKEN")
+                        )
                     elif endpoint == "ollama.com":
                         custom_llm_provider = "ollama"
                         dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
@@ -707,7 +715,7 @@ def _get_openai_compatible_provider_info(
             dynamic_api_key,
         ) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "together_ai":
-        api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.xyz/v1"
+        api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.ai/v1"
         dynamic_api_key = api_key or (
             get_secret_str("TOGETHER_API_KEY")
             or get_secret_str("TOGETHER_AI_API_KEY")

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -172,7 +172,7 @@ def get_supported_openai_params(
         if request_type == "embeddings":
             return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "together_ai":
-        return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
+        return litellm.TogetherAIChatConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "databricks":
         if request_type == "chat_completion":
             return litellm.DatabricksConfig().get_supported_openai_params(model=model)

litellm/litellm_core_utils/litellm_logging.py

@@ -62,7 +62,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.deepeval.deepeval import DeepEvalLogger
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
-from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
+from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     cost_breakdown_with_guardrail,
@@ -71,6 +71,9 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
+from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    InteractionsUsageObjectTransformation,
+)
 from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
 from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
 from litellm.litellm_core_utils.redact_messages import (
@@ -83,6 +86,10 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.responses.utils import ResponseAPILoggingUtils
 from litellm.types.agents import LiteLLMSendMessageResponse
 from litellm.types.containers.main import ContainerObject
+from litellm.types.interactions import (
+    InteractionsAPIResponse,
+    InteractionsAPIStreamingResponse,
+)
 from litellm.types.llms.openai import (
     AllMessageValues,
     Batch,
@@ -2145,6 +2152,11 @@ class Logging(LiteLLMLoggingBaseClass):
             or isinstance(logging_result, OpenAIModerationResponse)
             or isinstance(logging_result, OCRResponse)  # OCR
             or isinstance(logging_result, SearchResponse)  # Search API
+            or (
+                isinstance(logging_result, InteractionsAPIResponse)
+                and logging_result.usage is not None
+                and self._is_interactions_create_call_type()
+            )
             or isinstance(logging_result, dict)
             and logging_result.get("object") == "vector_store.search_results.page"
             or isinstance(logging_result, dict)
@@ -2157,6 +2169,87 @@ class Logging(LiteLLMLoggingBaseClass):
             return True
         return False
 
+    def _is_interactions_create_call_type(self) -> bool:
+        """
+        Only interaction creation is billable. GET polls, deletes, and cancels
+        also return an ``InteractionsAPIResponse`` (with usage once completed),
+        so recognizing those would write spend on every poll of a background
+        interaction. The proxy sets ``call_type`` from its route_type
+        (``create_interaction``/``acreate_interaction``); the SDK sets it from
+        the decorated function name (``create``/``acreate``).
+
+        Recognition additionally requires a usage block (checked at the call
+        site): a ``background=true`` create returns ``in_progress`` without
+        usage, and billing it would write a $0 spend log under the interaction
+        id that collides with the row the background poll task writes once the
+        interaction completes (see
+        ``litellm.interactions.background_cost_polling``).
+        """
+        return self.call_type in (
+            CallTypes.create_interaction.value,
+            CallTypes.acreate_interaction.value,
+            "create",
+            "acreate",
+        )
+
+    async def async_log_background_interaction_completion(
+        self,
+        result: InteractionsAPIResponse,
+    ) -> None:
+        """
+        Log the terminal result of a background interaction as a fresh success
+        event. The create request already ran success logging for its
+        ``in_progress`` response (no usage, so no cost was tracked); clearing
+        the dedup flags lets the completed result flow through cost calculation
+        and spend tracking exactly once, spanning create to completion.
+
+        The poll fetched this body through its own client call, which priced it
+        against a throwaway logging object holding none of this request's
+        deployment context: no ``model_info``, no router ``model_id``, no
+        deployment ``litellm_params``. Keeping that price would bill a
+        custom-priced deployment at the wrong rate, and it would also satisfy
+        the "already calculated" shortcut and skip repricing here, leaving the
+        cost breakdown at the zeros the usage-less create stamped and writing
+        those zeros to the spend log. Dropping it makes this event price the
+        settled body itself, against the deployment that served the create.
+
+        The same throwaway call stamped the deployment identity that travels
+        with the price, so ``model_id`` and ``litellm_model_name`` go with it.
+        Left in place they overwrite the create's real deployment with the
+        poll's empty one in the payload every logging integration reads.
+        """
+        settled_hidden_params: Final = getattr(result, "_hidden_params", None)
+        if isinstance(settled_hidden_params, dict):
+            for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
+                settled_hidden_params.pop(poll_scoped_key, None)
+        self._reset_success_emission_dedupe()
+        await self.async_success_handler(result=result)
+
+    def _reset_success_emission_dedupe(self) -> None:
+        """
+        Success callbacks dedupe per request, because the sync and async
+        handlers both fire on some paths and would otherwise report one call
+        twice. A settled background interaction is a genuinely second success
+        event on the same request, so every such marker has to be cleared or
+        the completion, the only event that carries usage and cost, is
+        discarded as a duplicate of the in-progress create.
+        """
+        self.model_call_details.pop("has_logged_async_success", None)
+        litellm_params = self.model_call_details.get("litellm_params")
+        if not isinstance(litellm_params, dict):
+            return
+        metadata = litellm_params.get("metadata")
+        if not isinstance(metadata, dict):
+            return
+        otel_internal = metadata.get("_otel_internal")
+        if not isinstance(otel_internal, dict):
+            return
+        spans_logged = otel_internal.get("spans_logged")
+        if not isinstance(spans_logged, dict):
+            return
+        for scope in [key for key in spans_logged if isinstance(key, tuple) and key[-1:] == ("success",)]:
+            del spans_logged[scope]
+
     def _flush_passthrough_collected_chunks_helper(
         self,
         raw_bytes: list[bytes],
@@ -2282,7 +2375,9 @@ class Logging(LiteLLMLoggingBaseClass):
         is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
         try:
             ## BUILD COMPLETE STREAMED RESPONSE
-            complete_streaming_response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None = None
+            complete_streaming_response: (
+                ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
+            ) = None
             if "complete_streaming_response" in self.model_call_details:
                 return  # break out of this.
             complete_streaming_response = self._get_assembled_streaming_response(
@@ -2768,14 +2863,14 @@ class Logging(LiteLLMLoggingBaseClass):
         ## BUILD COMPLETE STREAMED RESPONSE
         if "async_complete_streaming_response" in self.model_call_details:
             return  # break out of this.
-        complete_streaming_response: Final[ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None] = (
-            self._get_assembled_streaming_response(
-                result=result,
-                start_time=start_time,
-                end_time=end_time,
-                is_async=True,
-                streaming_chunks=self.streaming_chunks,
-            )
+        complete_streaming_response: Final[
+            ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
+        ] = self._get_assembled_streaming_response(
+            result=result,
+            start_time=start_time,
+            end_time=end_time,
+            is_async=True,
+            streaming_chunks=self.streaming_chunks,
         )
 
         if complete_streaming_response is not None:
@@ -3029,6 +3124,13 @@ class Logging(LiteLLMLoggingBaseClass):
         if not hasattr(self, "model_call_details"):
             self.model_call_details = {}
 
+        if (
+            self.model_call_details.get("log_event_type") == "failed_api_call"
+            and self.model_call_details.get("exception") is exception
+            and self.model_call_details.get("standard_logging_object") is not None
+        ):
+            return start_time, self.model_call_details["end_time"]
+
         self.model_call_details["log_event_type"] = "failed_api_call"
         self.model_call_details["exception"] = exception
         self.model_call_details["traceback_exception"] = (
@@ -3558,7 +3660,7 @@ class Logging(LiteLLMLoggingBaseClass):
         end_time: datetime.datetime,
         is_async: bool,
         streaming_chunks: list[object],
-    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None:
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
         if self.stream is not True:
             return None
         if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
@@ -3583,9 +3685,40 @@ class Logging(LiteLLMLoggingBaseClass):
                     ),
                 )
             return result.response
+        elif isinstance(result, InteractionsAPIStreamingResponse):
+            return self._assemble_completed_interaction_response(result)
         else:
             return None
 
+    @staticmethod
+    def _assemble_completed_interaction_response(
+        result: InteractionsAPIStreamingResponse,
+    ) -> InteractionsAPIResponse | None:
+        """
+        The Interactions API streaming iterator hands the terminal event to the
+        success handlers: the new schema (Api-Revision: 2026-05-20) emits
+        ``interaction.completed`` carrying the full interaction object, the
+        legacy schema (2026-05-07) emits a chunk with ``status="completed"``
+        and usage on the chunk itself. Build the equivalent non-streaming
+        response so cost calculation and spend tracking see one shape.
+        """
+        if result.event_type == "interaction.completed" and result.interaction is not None:
+            return InteractionsAPIResponse(**result.interaction)
+        if result.status == "completed":
+            return InteractionsAPIResponse(
+                **result.model_dump(
+                    exclude={  # mutable-ok: pydantic types exclude as set[str], which a frozenset does not satisfy
+                        "event_type",
+                        "delta",
+                        "index",
+                        "step",
+                        "interaction_id",
+                        "interaction",
+                    }
+                )
+            )
+        return None
+
     def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
         """
         Handles logging for Anthropic messages responses.
@@ -5092,6 +5225,8 @@ class StandardLoggingPayloadSetup:
         elif isinstance(usage, dict):
             if ResponseAPILoggingUtils._is_response_api_usage(usage):
                 return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
+            if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
+                return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
             return Usage(**usage)
 
         raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
@@ -5118,6 +5253,8 @@ class StandardLoggingPayloadSetup:
         if isinstance(_raw, dict):
             if ResponseAPILoggingUtils._is_response_api_usage(_raw):
                 return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
+            if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
+                return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
             return _raw
         if isinstance(_raw, Usage):
             return _raw.model_dump()
@@ -5325,9 +5462,10 @@ class StandardLoggingPayloadSetup:
         error_class: Final[str] = str(original_exception.__class__.__name__) if original_exception else ""
         _llm_provider_in_exception: Final = getattr(original_exception, "llm_provider", "")
 
-        # Get traceback information (first 100 lines)
         traceback_info = traceback_str or ""
-        if original_exception:
+        if original_exception and (
+            litellm.log_client_error_tracebacks or not is_expected_client_error(original_exception)
+        ):
             tb: Final[TracebackType | None] = getattr(original_exception, "__traceback__", None)
             if tb:
                 tb_lines: Final = traceback.format_tb(tb)
@@ -5800,11 +5938,15 @@ def get_standard_logging_object_payload(
             response_model_name = final_response_obj.get("model")
 
         # For Azure Model Router, preserve the actual model in the top-level standard
-        # logging payload only when the user has opted in.
+        # logging payload.
+        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
         requested_model: Final = kwargs.get("model")
-        if (
-            isinstance(requested_model, str)
-            and ("model_router" in requested_model.lower() or "model-router" in requested_model.lower())
+        stamped_selected_model: Final = AzureFoundryModelInfo.get_model_router_selected_model(hidden_params)
+        if stamped_selected_model is not None:
+            model_name = stamped_selected_model
+        elif (
+            AzureFoundryModelInfo.is_model_router_call(model=requested_model, hidden_params=hidden_params)
             and isinstance(response_model_name, str)
             and response_model_name
         ):

litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py

@@ -1,6 +1,9 @@
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import Any
 
 from litellm.types.utils import (
+    CompletionTokensDetailsWrapper,
     PromptTokensDetailsWrapper,
     TranscriptionUsageDurationObject,
     TranscriptionUsageTokensObject,
@@ -34,3 +37,127 @@ class TranscriptionUsageObjectTransformation:
                 ),
             )
         return None
+
+
+_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
+    {
+        "text": "text_tokens",
+        "audio": "audio_tokens",
+        "image": "image_tokens",
+        "video": "video_tokens",
+        "document": "text_tokens",
+    }
+)
+
+
+def _modality_field(entry: Mapping[str, Any]) -> str | None:
+    return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
+
+
+def _token_count(value: object) -> int:
+    return value if isinstance(value, int) else 0
+
+
+def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
+    fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
+    return MappingProxyType(
+        {
+            field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
+            for field in fields
+        }
+    )
+
+
+def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
+    return sum(
+        _token_count(entry.get("count"))
+        for entry in tuple(usage_object.get("grounding_tool_count") or ())
+        if isinstance(entry, Mapping) and entry.get("type") == "google_search"  # pyright: ignore[reportUnnecessaryIsInstance]  # provider JSON, not the empty tuple inferred from `or ()`
+    )
+
+
+def _subtract_cached_from_input(
+    input_sums: Mapping[str, int],
+    cached_sums: Mapping[str, int],
+    total_cached_tokens: int,
+) -> Mapping[str, int]:
+    if cached_sums:
+        return MappingProxyType(
+            {field: max(0, tokens - cached_sums.get(field, 0)) for field, tokens in input_sums.items()}
+        )
+    if total_cached_tokens and "text_tokens" in input_sums:
+        return MappingProxyType(
+            {
+                **input_sums,
+                "text_tokens": max(0, input_sums["text_tokens"] - total_cached_tokens),
+            }
+        )
+    return input_sums
+
+
+class InteractionsUsageObjectTransformation:
+    """
+    Maps the Google Interactions API usage block (total_input_tokens,
+    output_tokens_by_modality, ...) into LiteLLM's chat-format ``Usage`` so the
+    generic cost calculator and spend tracking can bill it.
+    """
+
+    @staticmethod
+    def is_interactions_usage_object(usage_object: object) -> bool:
+        if not isinstance(usage_object, dict):
+            return False
+        if "prompt_tokens" in usage_object or "input_tokens" in usage_object:
+            return False
+        return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
+
+    @staticmethod
+    def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
+        input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
+            usage_object.get("tool_use_tokens_by_modality") or ()
+        )
+        cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
+        output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
+
+        total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
+        input_sums = _subtract_cached_from_input(
+            input_sums=_modality_token_sums(input_entries),
+            cached_sums=cached_sums,
+            total_cached_tokens=total_cached_tokens,
+        )
+
+        reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
+            usage_object.get("total_thought_tokens")
+        )
+        prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
+            usage_object.get("total_tool_use_tokens")
+        )
+        completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
+        total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
+
+        web_search_requests = _google_search_query_count(usage_object)
+        prompt_tokens_details = (
+            PromptTokensDetailsWrapper(
+                cached_tokens=total_cached_tokens or None,
+                web_search_requests=web_search_requests or None,
+                **input_sums,
+            )
+            if input_sums or total_cached_tokens or web_search_requests
+            else None
+        )
+        completion_tokens_details = (
+            CompletionTokensDetailsWrapper(
+                reasoning_tokens=reasoning_tokens or None,
+                **output_sums,
+            )
+            if output_sums or reasoning_tokens
+            else None
+        )
+
+        return Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            prompt_tokens_details=prompt_tokens_details,
+            completion_tokens_details=completion_tokens_details,
+            cache_read_input_tokens=total_cached_tokens or None,
+        )

litellm/litellm_core_utils/llm_request_utils.py

@@ -1,8 +1,88 @@
+from collections.abc import Mapping
 from typing import Final
 
 import litellm
 
 
+def _form_field_value(value: object) -> str:
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    return str(value)
+
+
+def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]:
+    if isinstance(value, Mapping):
+        return tuple(
+            item for subkey, subvalue in value.items() for item in _flatten_form_field(f"{key}[{subkey}]", subvalue)
+        )
+    if isinstance(value, (list, tuple)):
+        return tuple(item for entry in value for item in _flatten_form_field(f"{key}[]", entry))
+    if value is None:
+        return ()
+    serialized: Final = _form_field_value(value)
+    if not serialized:
+        return ()
+    return ((key, serialized),)
+
+
+def _is_form_scalar(value: object) -> bool:
+    return value is not None and not isinstance(value, (Mapping, list, tuple))
+
+
+def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
+    if isinstance(value, Mapping):
+        return tuple(
+            item
+            for subkey, subvalue in value.items()
+            for item in _flatten_form_data_field(f"{key}[{subkey}]", subvalue)
+        )
+    if isinstance(value, (list, tuple)):
+        if all(_is_form_scalar(entry) for entry in value):
+            serialized_fields: Final = tuple(field for entry in value if (field := _form_field_value(entry)))
+            return ((key, serialized_fields),) if serialized_fields else ()
+        return tuple(item for entry in value for item in _flatten_form_data_field(f"{key}[]", entry))
+    if value is None:
+        return ()
+    serialized: Final = _form_field_value(value)
+    if not serialized:
+        return ()
+    return ((key, serialized),)
+
+
+def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
+    """
+    Flatten JSON-shaped bodies into ``(name, value)`` form fields for a ``dict``-backed
+    multipart body, applying ``sources`` in order so a later source wins on a key collision
+    under ``dict.update``. Nested objects become ``key[subkey]`` fields the way the OpenAI SDK
+    serializes them, so provider params reach a multipart request without handing the httpx
+    encoder a nested value it rejects with ``Invalid type for value``. A scalar list becomes a
+    single field carrying a tuple value, which httpx emits as one repeated part per element, so
+    every element survives instead of collapsing to the last under ``dict.update``.
+    """
+    return tuple(
+        pair
+        for source in sources
+        if source is not None
+        for top_key, top_value in source.items()
+        for pair in _flatten_form_data_field(top_key, top_value)
+    )
+
+
+def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]:
+    """
+    Encode a JSON-shaped body as OpenAI-SDK-style multipart file-tuples so a file-less
+    request is still sent as multipart/form-data, working around httpx downgrading a
+    file-less ``data=`` payload to application/x-www-form-urlencoded.
+    """
+    return tuple(
+        (key, (None, serialized))
+        for top_key, top_value in data.items()
+        for key, serialized in _flatten_form_field(top_key, top_value)
+    )
+
+
 def _ensure_extra_body_is_safe(extra_body: dict | None) -> dict | None:
     """
     Ensure that the extra_body sent in the request is safe,  otherwise users will see this error

litellm/litellm_core_utils/logging_worker.py

@@ -5,7 +5,7 @@ import asyncio
 import atexit
 import contextvars
 import logging
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Iterator
 from typing import Final
 
 from typing_extensions import TypedDict
@@ -61,6 +61,19 @@ class LoggingWorker:
         # Register cleanup handler to flush remaining events on exit
         atexit.register(self._flush_on_exit)
 
+    @staticmethod
+    def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
+        """Pop every task still queued, without awaiting them, so they can be moved to another queue."""
+
+        def _pop_until_empty() -> Iterator[LoggingTask]:
+            while True:
+                try:
+                    yield queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    return
+
+        return tuple(_pop_until_empty())
+
     def _ensure_queue(self) -> None:
         """Initialize the queue if it doesn't exist or if event loop has changed."""
         try:
@@ -69,14 +82,27 @@ class LoggingWorker:
             # No running loop, can't initialize
             return
 
-        # Check if we need to reinitialize due to event loop change
+        # The queue, semaphore and worker task are all bound to the loop that created them. On a
+        # loop change we hand the still-pending tasks to a fresh queue instead of dropping them,
+        # so queued spend-logging coroutines are not silently discarded (and never left un-awaited).
         if self._queue is not None and self._bound_loop is not current_loop:
-            verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
-            # Clear old state - these are bound to the old loop
-            self._queue = None
+            carried_over: Final = self._drain_pending(self._queue)
+            new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
+            for carried_task in carried_over:
+                new_queue.put_nowait(carried_task)
+            if carried_over:
+                verbose_logger.warning(
+                    "LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
+                    len(carried_over),
+                )
+            else:
+                verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
             self._sem = None
             self._worker_task = None
             self._running_tasks.clear()
+            self._queue = new_queue
+            self._bound_loop = current_loop
+            return
 
         if self._queue is None:
             self._queue = asyncio.Queue(maxsize=self.max_queue_size)

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -28,8 +28,12 @@ from litellm.types.llms.openai import (
     ChatCompletionAssistantMessage,
     ChatCompletionFileObject,
     ChatCompletionImageObject,
+    ChatCompletionReasoningItem,
+    ChatCompletionReasoningSummaryTextBlock,
+    ChatCompletionRedactedThinkingBlock,
     ChatCompletionResponseMessage,
     ChatCompletionTextObject,
+    ChatCompletionThinkingBlock,
     ChatCompletionToolParam,
     ChatCompletionUserMessage,
 )
@@ -466,6 +470,8 @@ def update_messages_with_model_file_ids(
     from litellm.proxy.openai_files_endpoints.common_utils import (
         _is_base64_encoded_unified_file_id,
         convert_b64_uid_to_unified_uid,
+        get_original_file_id,
+        is_model_embedded_id,
     )
 
     for message in messages:
@@ -504,6 +510,11 @@ def update_messages_with_model_file_ids(
                                 unified_file_id = convert_b64_uid_to_unified_uid(file_id)
                                 if "llm_output_file_id," in unified_file_id:
                                     provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
+                            if not provider_file_id and is_model_embedded_id(file_id):
+                                # `litellm:<raw_id>;model,<m>` encoding from the
+                                # x-litellm-model upload path. Strip the wrapper
+                                # so the provider sees its own ID.
+                                provider_file_id = get_original_file_id(file_id)
                             file_object_file_field["file_id"] = provider_file_id or file_id
                         if format:
                             file_object_file_field["format"] = format
@@ -531,6 +542,8 @@ def update_responses_input_with_model_file_ids(
     from litellm.proxy.openai_files_endpoints.common_utils import (
         _is_base64_encoded_unified_file_id,
         convert_b64_uid_to_unified_uid,
+        get_original_file_id,
+        is_model_embedded_id,
     )
 
     if isinstance(input, str):
@@ -574,6 +587,13 @@ def update_responses_input_with_model_file_ids(
                                 updated_content_item = content_item.copy()
                                 updated_content_item["file_id"] = provider_file_id
                                 updated_content.append(updated_content_item)
+                            elif is_model_embedded_id(file_id):
+                                # `litellm:<raw_id>;model,<m>` encoding from the
+                                # x-litellm-model upload path. Strip the wrapper
+                                # so the provider sees its own ID.
+                                updated_content_item = content_item.copy()
+                                updated_content_item["file_id"] = get_original_file_id(file_id)
+                                updated_content.append(updated_content_item)
                             else:
                                 # Not a managed file, keep as-is
                                 updated_content.append(content_item)
@@ -1549,6 +1569,44 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
     return None, message_content
 
 
+def _readable_thinking_text(
+    block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
+) -> str:
+    """The text a chat model can read back, empty for redacted blocks and malformed ones."""
+    if block.get("type") != "thinking":
+        return ""
+    thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking")  # cast-ok: narrowed by the type tag
+    return str(thinking or "")
+
+
+def reasoning_content_from_thinking_blocks(
+    thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
+) -> str:
+    """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
+
+    Redacted blocks carry no readable text, so they contribute nothing.
+    """
+    return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
+
+
+def responses_reasoning_item_from_thinking_blocks(
+    thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
+) -> ChatCompletionReasoningItem | None:
+    """Build a Responses API `reasoning` input item from Anthropic thinking blocks.
+
+    The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
+    did not mint itself, while an item without an id is always accepted.
+    """
+    summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [  # mutable-ok: API message payload
+        ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
+        for block in thinking_blocks
+        if (text := _readable_thinking_text(block))
+    ]
+    if not summary:
+        return None
+    return ChatCompletionReasoningItem(type="reasoning", summary=summary)
+
+
 def _parse_content_for_reasoning(
     message_text: str | None,
 ) -> tuple[str | None, str | None]:

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -16,6 +16,7 @@ import litellm.types
 import litellm.types.llms
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.constants import REDACTED_BY_LITELLM
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
 from litellm.types.files import get_file_extension_from_mime_type
@@ -642,49 +643,6 @@ def claude_2_1_pt(
     return prompt
 
 
-### TOGETHER AI
-
-
-def get_model_info(token, model):
-    try:
-        headers: Final = {"Authorization": f"Bearer {token}"}
-        client: Final = HTTPHandler(concurrent_limit=1)
-        response: Final = client.get("https://api.together.xyz/models/info", headers=headers)
-        if response.status_code == 200:
-            model_info: Final = response.json()
-            for m in model_info:
-                if m["name"].lower().strip() == model.strip():
-                    return m["config"].get("prompt_format", None), m["config"].get("chat_template", None)
-            return None, None
-        else:
-            return None, None
-    except Exception:  # safely fail a prompt template request
-        return None, None
-
-
-## OLD TOGETHER AI FLOW
-# def format_prompt_togetherai(messages, prompt_format, chat_template):
-#     if prompt_format is None:
-#         return default_pt(messages)
-
-#     human_prompt, assistant_prompt = prompt_format.split("{prompt}")
-
-#     if chat_template is not None:
-#         prompt = hf_chat_template(
-#             model=None, messages=messages, chat_template=chat_template
-#         )
-#     elif prompt_format is not None:
-#         prompt = custom_prompt(
-#             role_dict={},
-#             messages=messages,
-#             initial_prompt_value=human_prompt,
-#             final_prompt_value=assistant_prompt,
-#         )
-#     else:
-#         prompt = default_pt(messages)
-#     return prompt
-
-
 ### IBM Granite
 
 
@@ -5383,12 +5341,13 @@ def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) ->
         return raw
     if not isinstance(raw, str):
         return {}
+    normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
     from litellm.litellm_core_utils.prompt_templates.common_utils import (
         parse_tool_call_arguments,
     )
 
     try:
-        parsed: Final = parse_tool_call_arguments(raw, tool_name=tool_name, context=context)
+        parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
     except ValueError as e:
         verbose_logger.warning("Failed to parse tool call arguments: %s", e)
         return {}

litellm/litellm_core_utils/redact_messages.py

@@ -13,6 +13,7 @@ import inspect
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
+from litellm.constants import REDACTED_BY_LITELLM
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
@@ -84,29 +85,29 @@ def _redact_tool_calls(tool_calls) -> None:
     for tool_call in tool_calls:
         function = getattr(tool_call, "function", None)
         if function is not None and hasattr(function, "arguments"):
-            function.arguments = "redacted-by-litellm"
+            function.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_function_call(function_call) -> None:
     """Redact legacy assistant function_call arguments."""
     if function_call is not None and hasattr(function_call, "arguments"):
-        function_call.arguments = "redacted-by-litellm"
+        function_call.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
     if isinstance(choice, litellm.Choices):
-        choice.message.content = "redacted-by-litellm"
+        choice.message.content = REDACTED_BY_LITELLM
         if hasattr(choice.message, "reasoning_content"):
-            choice.message.reasoning_content = "redacted-by-litellm"
+            choice.message.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
         _redact_tool_calls(getattr(choice.message, "tool_calls", None))
         _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
-        choice.delta.content = "redacted-by-litellm"
+        choice.delta.content = REDACTED_BY_LITELLM
         if hasattr(choice.delta, "reasoning_content"):
-            choice.delta.reasoning_content = "redacted-by-litellm"
+            choice.delta.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
         _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
@@ -117,22 +118,22 @@ def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
         if hasattr(output_item, "text"):
-            output_item.text = "redacted-by-litellm"
+            output_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "content") and isinstance(output_item.content, list):
             for content_part in output_item.content:
                 if hasattr(content_part, "text"):
-                    content_part.text = "redacted-by-litellm"
+                    content_part.text = REDACTED_BY_LITELLM
 
         # Redact reasoning items in output array
         if hasattr(output_item, "type") and output_item.type == "reasoning":
             if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
                 for summary_item in output_item.summary:
                     if hasattr(summary_item, "text"):
-                        summary_item.text = "redacted-by-litellm"
+                        summary_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
-            output_item.arguments = "redacted-by-litellm"
+            output_item.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_responses_api_output_dict(output_items, redacted_str: str):
@@ -164,7 +165,7 @@ def _redact_standard_logging_object(model_call_details: dict):
     if standard_logging_object is None:
         return
 
-    redacted_str: Final = "redacted-by-litellm"
+    redacted_str: Final = REDACTED_BY_LITELLM
 
     if standard_logging_object.get("messages") is not None:
         standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
@@ -235,7 +236,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
     copy via redact_streaming_responses_for_custom_logger instead.
     """
     # Redact model_call_details
-    model_call_details["messages"] = [{"role": "user", "content": "redacted-by-litellm"}]
+    model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""
     _redact_standard_logging_object(model_call_details)
@@ -256,7 +257,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             or hasattr(result, "__anext__")  # async generator
         ):  # async iterator
             # For async objects, return a simple redacted response without deepcopy
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
 
         if not (
             isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
@@ -273,11 +274,11 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
         elif isinstance(_result, dict) and "choices" in _result:
             # Handle dict representation of ModelResponse (e.g., from model_dump())
             if _result.get("choices") is not None:
-                _redact_model_response_dict_choices(_result["choices"], "redacted-by-litellm")
+                _redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
             redact_vertex_ai_metadata_from_logged_object(_result)
         elif isinstance(_result, dict) and "output" in _result:
             if isinstance(_result.get("output"), list):
-                _redact_responses_api_output_dict(_result["output"], "redacted-by-litellm")
+                _redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
         elif isinstance(_result, litellm.ResponsesAPIResponse):
             if hasattr(_result, "output"):
                 _redact_responses_api_output(_result.output)
@@ -288,7 +289,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             if hasattr(_result, "data") and _result.data is not None:
                 _result.data = []
         else:
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
         return _result
 
 

litellm/llms/anthropic/chat/transformation.py

@@ -1,7 +1,8 @@
 import json
 import re
 import time
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, NoReturn, cast
 
 import httpx
@@ -121,6 +122,32 @@ else:
 # response side.
 _ANTHROPIC_TOOL_NAME_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
 _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
+
+_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[Any], bool]]] = MappingProxyType(
+    {
+        "null": lambda v: v is None,
+        "boolean": lambda v: isinstance(v, bool),
+        "integer": lambda v: isinstance(v, int) and not isinstance(v, bool),
+        "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
+        "string": lambda v: isinstance(v, str),
+        "array": lambda v: isinstance(v, list),
+        "object": lambda v: isinstance(v, dict),
+    }
+)
+
+
+def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool:
+    """Whether ``schema``'s ``enum`` cannot match its declared ``type``."""
+    enum_values: Final = schema.get("enum")
+    declared_type: Final = schema.get("type")
+    if not isinstance(enum_values, list) or declared_type is None:
+        return False
+    if isinstance(declared_type, list):
+        return True
+    check: Final = _ENUM_TYPE_CHECKS.get(declared_type)
+    return check is not None and not all(check(value) for value in enum_values)
+
+
 # Single, internal-only key on ``litellm_params`` used to thread the per-
 # request reverse map (sanitized -> original) from request build to response
 # parsing. ``litellm_params`` is never serialized to a provider; ``optional_
@@ -565,9 +592,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             else:
                 result["description"] = constraint_note
 
+        drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema)
+
         for key, value in schema.items():
             if key in unsupported_fields:
                 continue
+            if key == "type" and drops_conflicting_type:
+                continue
             if key == "description" and "description" in result:
                 # Already handled above
                 continue
@@ -1184,8 +1215,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if reasoning_effort is None or reasoning_effort == "none":
             return None
         if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
+            # without display, Anthropic defaults adaptive thinking to
+            # display="omitted" and returns a blank thinking block
             return AnthropicThinkingParam(
                 type="adaptive",
+                display="summarized",
             )
         elif reasoning_effort == "low":
             return AnthropicThinkingParam(
@@ -2113,7 +2147,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         )
 
     @staticmethod
-    def _thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None:
+    def thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None:
         details: Final = usage_object.get("output_tokens_details")
         if not isinstance(details, Mapping):
             return None
@@ -2145,7 +2179,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         reported_thinking_tokens: Final = (
             iteration_thinking_tokens
             if iteration_thinking_tokens is not None
-            else self._thinking_tokens_from_usage(usage_object)
+            else self.thinking_tokens_from_usage(usage_object)
         )
         if reported_thinking_tokens is not None:
             capped_reported: Final = min(max(0, reported_thinking_tokens), completion_tokens)
@@ -2168,7 +2202,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def _sum_iteration_thinking_tokens(self, iterations: Sequence[object]) -> int | None:
         per_iteration: Final = tuple(
-            self._thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None
+            self.thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None
             for iteration in iterations
         )
         reported: Final = tuple(tokens for tokens in per_iteration if tokens is not None)

litellm/llms/anthropic/common_utils.py

@@ -38,6 +38,21 @@ DROP_DISABLED_THINKING_WARNING: Final = (
     "thinking blocks, and those thinking tokens are billed as output tokens."
 )
 
+# Anthropic error `type` (both the JSON error body and SSE `event: error`
+# payloads use this field) mapped to the HTTP status code it corresponds to.
+ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = MappingProxyType(
+    {
+        "invalid_request_error": 400,
+        "authentication_error": 401,
+        "permission_error": 403,
+        "not_found_error": 404,
+        "rate_limit_error": 429,
+        "api_error": 500,
+        "overloaded_error": 503,
+        "timeout_error": 504,
+    }
+)
+
 _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
@@ -440,6 +455,16 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         """
         return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
 
+    @staticmethod
+    def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool:
+        """Whether ``model`` is an adaptive-thinking model that still accepts legacy
+        ``thinking.type=enabled`` with ``budget_tokens`` (the Claude 4.6 family).
+        The model cost map is authoritative: an explicit ``supports_legacy_thinking``
+        entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations``
+        rule for unmapped 4.6 ids. Absent flag means the model rejects the legacy shape.
+        """
+        return AnthropicModelInfo._supports_model_capability(model, "supports_legacy_thinking", custom_llm_provider)
+
     @staticmethod
     def maybe_drop_disabled_thinking(
         model: str,

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -64,6 +64,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
 
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     parse_tool_call_arguments,
+    reasoning_content_from_thinking_blocks,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -592,6 +593,9 @@ class LiteLLMAnthropicMessagesAdapter:
                     assistant_message["tool_calls"] = tool_calls
                 if len(thinking_blocks) > 0:
                     assistant_message["thinking_blocks"] = thinking_blocks
+                reasoning_content = reasoning_content_from_thinking_blocks(thinking_blocks)
+                if reasoning_content:
+                    assistant_message["reasoning_content"] = reasoning_content
                 new_messages.append(assistant_message)
 
         return new_messages

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -1,6 +1,6 @@
 import asyncio
 import json
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from datetime import datetime
 from typing import Any, Final, Protocol, runtime_checkable
 
@@ -11,9 +11,11 @@ from typing_extensions import TypedDict
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
 from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
 from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
@@ -33,26 +35,239 @@ def _is_message_stop_chunk(chunk: object) -> bool:
     return False
 
 
-def _is_provider_error_chunk(chunk: object) -> bool:
+def is_anthropic_ping_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk is a pure ``ping`` keepalive frame. It carries no content
+    and can recur indefinitely on a slow-starting or idle connection, so a
+    mid-stream fallback wrapper drops it outright while still deciding
+    whether to commit to the primary stream, rather than buffering it.
+
+    A physical transport chunk that coalesces a ping with any other SSE
+    event (``message_start``, ``content_block_delta``, ``event: error``, ...)
+    is NOT a pure ping - dropping it whole would discard those events - so
+    only a chunk whose every ``event:`` line is ``event: ping`` qualifies.
+    """
     if isinstance(chunk, dict):
-        return chunk.get("type") == "error"
+        return chunk.get("type") == "ping"
     if isinstance(chunk, (bytes, bytearray)):
-        return any(line == b"event: error" for line in chunk.splitlines())
+        event_lines: Final = tuple(line for line in chunk.splitlines() if line.startswith(b"event:"))
+        return bool(event_lines) and all(line == b"event: ping" for line in event_lines)
     return False
 
 
+def is_anthropic_content_delta_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk carries actual assistant-generated output (a
+    ``content_block_delta`` frame), as opposed to a lifecycle/bookkeeping
+    frame (``message_start``, ``content_block_start``/``stop``,
+    ``message_delta``, ``message_stop``, ``ping``) that carries nothing
+    worth preserving before an invisible mid-stream fallback retry.
+    """
+    if isinstance(chunk, dict):
+        return chunk.get("type") == "content_block_delta"
+    if isinstance(chunk, (bytes, bytearray)):
+        return any(line == b"event: content_block_delta" for line in chunk.splitlines())
+    return False
+
+
+def _decoded_sse_data_line(line: bytes) -> object | None:
+    if not line.startswith(b"data:"):
+        return None
+    try:
+        return json.loads(line[len(b"data:") :].strip())
+    except (ValueError, TypeError):
+        return None
+
+
+def _anthropic_error_event_payload(chunk: object) -> Mapping[str, object] | None:
+    if isinstance(chunk, dict):
+        return chunk if chunk.get("type") == "error" else None
+    if isinstance(chunk, (bytes, bytearray)):
+        decoded_lines: Final = (_decoded_sse_data_line(line) for line in chunk.splitlines())
+        return next(
+            (
+                candidate
+                for candidate in decoded_lines
+                if isinstance(candidate, dict) and candidate.get("type") == "error"
+            ),
+            None,
+        )
+    return None
+
+
+def _anthropic_error_body(chunk: object) -> Mapping[str, object] | None:
+    """Return the ``error`` object of an Anthropic SSE ``event: error`` chunk, or None."""
+    payload: Final = _anthropic_error_event_payload(chunk)
+    error_body: Final = payload.get("error") if payload is not None else None
+    return error_body if isinstance(error_body, dict) else None
+
+
+def _is_provider_error_chunk(chunk: object) -> bool:
+    return _anthropic_error_body(chunk) is not None
+
+
+def parse_anthropic_error_event(chunk: object) -> tuple[str, str, int] | None:
+    """
+    Extract ``(error_type, message, http_status_code)`` from an Anthropic SSE
+    ``event: error`` chunk (raw bytes or an already-decoded dict), or None if
+    ``chunk`` is not an error event.
+
+    The status code is looked up via ANTHROPIC_ERROR_STATUS_CODE_MAP,
+    defaulting to 500 for an error ``type`` Anthropic hasn't documented yet.
+    """
+    error_body: Final = _anthropic_error_body(chunk)
+    if error_body is None:
+        return None
+    error_type: Final = error_body.get("type")
+    if not isinstance(error_type, str):
+        return None
+    message: Final = error_body.get("message")
+    return (
+        error_type,
+        message if isinstance(message, str) else error_type,
+        ANTHROPIC_ERROR_STATUS_CODE_MAP.get(error_type, 500),
+    )
+
+
 def _is_terminal_stream_chunk(chunk: object) -> bool:
     return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
 
 
+def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
+    return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
 def _incomplete_stream_error_sse_event() -> bytes:
-    payload: Final = json.dumps(
-        {
-            "type": "error",
-            "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
-        }
+    return _sse_event(  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "error",
+        {"type": "error", "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE}},
+    )
+
+
+def _anthropic_content_block_start_and_deltas(
+    block: Mapping[str, object],
+) -> tuple[Mapping[str, object], tuple[Mapping[str, object], ...]]:
+    """
+    ``(content_block_start.content_block, content_block_delta.delta events)``
+    for one Anthropic response content block. A thinking block emits both a
+    thinking_delta and a trailing signature_delta - a real Anthropic stream
+    does the same, and dropping the signature makes any replay of that
+    assistant message (a follow-up turn, a tool-use continuation) fail
+    Anthropic's thinking-signature verification. redacted_thinking has no
+    delta at all - it is sent complete in content_block_start.
+    """
+    match block.get("type"):
+        case "tool_use":
+            return (
+                {  # mutable-ok: one-shot payload
+                    "id": block.get("id"),
+                    "name": block.get("name"),
+                    "input": {},  # mutable-ok: one-shot payload
+                    "type": "tool_use",
+                },
+                (
+                    {  # mutable-ok: one-shot payload
+                        "partial_json": json.dumps(block.get("input") or {}),  # mutable-ok: one-shot payload
+                        "type": "input_json_delta",
+                    },
+                ),
+            )
+        case "thinking":
+            signature: Final = block.get("signature")
+            signature_deltas: Final = (
+                ({"signature": signature, "type": "signature_delta"},)  # mutable-ok: one-shot payload
+                if isinstance(signature, str) and signature
+                else ()
+            )
+            return (
+                {"thinking": "", "signature": "", "type": "thinking"},  # mutable-ok: one-shot payload
+                (
+                    {"thinking": block.get("thinking") or "", "type": "thinking_delta"},  # mutable-ok: one-shot payload
+                    *signature_deltas,
+                ),
+            )
+        case "redacted_thinking":
+            return ({"type": "redacted_thinking", "data": block.get("data")}, ())  # mutable-ok: one-shot JSON payload
+        case _:
+            return (
+                {"type": "text", "text": ""},  # mutable-ok: one-shot JSON payload
+                ({"type": "text_delta", "text": block.get("text") or ""},),  # mutable-ok: one-shot JSON payload
+            )
+
+
+def anthropic_messages_response_as_sse_events(response: AnthropicMessagesResponse) -> tuple[bytes, ...]:
+    """
+    Render a complete (non-streaming) AnthropicMessagesResponse as the SSE
+    event sequence a real streaming request would have produced.
+
+    A mid-stream fallback can resolve to a non-streaming response even
+    though the client asked to stream (e.g. an agentic tool-use loop that
+    intercepts and returns a complete message) - yielding that dict directly
+    into a `/v1/messages` SSE byte stream would produce a malformed
+    response, so it's synthesized into the message_start/content_block_*/
+    message_delta/message_stop lifecycle a real stream would have sent.
+    """
+    content_blocks: Final = response.get("content") or ()
+    content_events: Final = (
+        event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
+    )
+    # A real message_start always carries a null stop_reason/stop_sequence and
+    # a zero output_tokens - those are only known once generation finishes, so
+    # copying the completed response's final values here would let a client
+    # treat the message as already finished, or double-count output tokens.
+    message_start_usage: Final = {  # mutable-ok: one-shot JSON payload
+        **(response.get("usage") or {}),
+        "output_tokens": 0,
+    }
+    message_start_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_start",
+        "message": {  # mutable-ok: one-shot JSON payload
+            **response,
+            "content": [],  # mutable-ok: one-shot JSON payload
+            "stop_reason": None,
+            "stop_sequence": None,
+            "usage": message_start_usage,
+        },
+    }
+    message_delta_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_delta",
+        "delta": {  # mutable-ok: one-shot JSON payload
+            "stop_reason": response.get("stop_reason"),
+            "stop_sequence": response.get("stop_sequence"),
+        },
+        "usage": response.get("usage") or {},  # mutable-ok: one-shot JSON payload
+    }
+    return (
+        _sse_event("message_start", message_start_payload),
+        *content_events,
+        _sse_event("message_delta", message_delta_payload),
+        _sse_event("message_stop", {"type": "message_stop"}),  # mutable-ok: one-shot JSON payload
+    )
+
+
+def _anthropic_content_block_events(index: int, block: Mapping[str, object]) -> tuple[bytes, ...]:
+    start_block, deltas = _anthropic_content_block_start_and_deltas(block)
+    start_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_start",
+        "index": index,
+        "content_block": start_block,
+    }
+    stop_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_stop",
+        "index": index,
+    }
+    delta_events: Final = tuple(
+        _sse_event(
+            "content_block_delta",
+            {"type": "content_block_delta", "index": index, "delta": delta},  # mutable-ok: one-shot payload
+        )
+        for delta in deltas
+    )
+    return (
+        _sse_event("content_block_start", start_payload),
+        *delta_events,
+        _sse_event("content_block_stop", stop_payload),
     )
-    return f"event: error\ndata: {payload}\n\n".encode()
 
 
 class AnthropicMessagesStreamHiddenParams(TypedDict):

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -379,13 +379,19 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     def _translate_legacy_thinking_for_adaptive_model(
         model: str, optional_params: dict, custom_llm_provider: str
     ) -> None:
-        """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7.
-        Caller-provided ``output_config.effort`` is never overridden.
+        """Translate legacy ``thinking.type=enabled`` to adaptive for the
+        adaptive-thinking models that reject it (4.7+ and the 5 families).
+        Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
+        legacy shape natively, so it is forwarded verbatim and the caller's
+        ``budget_tokens`` cap keeps applying. Caller-provided
+        ``output_config.effort`` is never overridden.
         """
         from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 
         if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
             return
+        if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
+            return
         thinking: Final = optional_params.get("thinking")
         if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
             return

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -152,7 +152,10 @@ class AnthropicResponsesStreamWrapper:
             if block_idx < 0:
                 if not delta:
                     return
-                block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""})
+                block_idx = self._open_block(
+                    item_id,
+                    {"type": "thinking", "thinking": "", "signature": ""},  # mutable-ok: API message payload
+                )
             self._chunk_queue.append(
                 {
                     "type": "content_block_delta",

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -6,12 +6,14 @@ path used for OpenAI and Azure models.
 """
 
 import json
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
+from itertools import groupby
 from typing import Any, Final, cast
 
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     TOOL_RESULT_IMAGE_BOUNDARY,
     TOOL_RESULT_IMAGE_PLACEHOLDER,
+    responses_reasoning_item_from_thinking_blocks,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import (
@@ -36,7 +38,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
     AnthropicUsage,
 )
-from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
+from litellm.types.llms.openai import (
+    ChatCompletionThinkingBlock,
+    ResponseAPIUsage,
+    ResponsesAPIResponse,
+)
 
 
 class LiteLLMAnthropicToResponsesAPIAdapter:
@@ -100,6 +106,58 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text"))  # pyright: ignore[reportUnnecessaryIsInstance]  # untrusted client payload
         ]
 
+    @staticmethod
+    def _summary_part_text(part: object) -> str:
+        if isinstance(part, Mapping):
+            mapping: Final = cast(Mapping[str, Any], part)  # cast-ok: summary parts are untyped provider json
+            return str(mapping.get("text") or "")
+        return str(getattr(part, "text", None) or "")
+
+    @classmethod
+    def _thinking_blocks_from_reasoning_item(
+        cls,
+        summary: Iterable[object],
+    ) -> tuple[dict[str, Any], ...]:  # mutable-ok: API message payload
+        """Anthropic thinking blocks for one Responses reasoning item.
+
+        The signature stays empty: only Anthropic can sign a thinking block, and a stand-in
+        value would be replayed as a real one and rejected by every backend that verifies it.
+        """
+        return tuple(
+            AnthropicResponseContentBlockThinking(
+                type="thinking",
+                thinking=text,
+                signature=None,
+            ).model_dump()
+            for part in summary
+            if (text := cls._summary_part_text(part))
+        )
+
+    @staticmethod
+    def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str:
+        """Group a run of consecutive thinking blocks together; keep every other block alone."""
+        index, block = indexed_block
+        return "thinking" if block.get("type") == "thinking" else f"block:{index}"
+
+    @classmethod
+    def _assistant_group_to_input_item(
+        cls, group: tuple[Mapping[str, Any], ...]
+    ) -> dict[str, Any] | None:  # mutable-ok: API message payload
+        first: Final = group[0]
+        btype: Final = first.get("type")
+        if btype == "thinking":
+            blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group)  # cast-ok: untrusted client payload
+            reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks)
+            return None if reasoning_item is None else dict(reasoning_item)  # mutable-ok: API message payload
+        if btype == "tool_use":
+            return {  # mutable-ok: API message payload
+                "type": "function_call",
+                "call_id": first.get("id", ""),
+                "name": first.get("name", ""),
+                "arguments": json.dumps(first.get("input", {})),  # mutable-ok: API message payload
+            }
+        return None
+
     def translate_messages_to_responses_input(
         self,
         messages: list[AllAnthropicPassThroughMessageValues],
@@ -113,6 +171,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
           user image         -> message(role=user, input_image)
           user tool_result   -> function_call_output
           assistant text     -> message(role=assistant, output_text)
+          assistant thinking -> reasoning
           assistant tool_use -> function_call
         """
         input_items: Final[list[dict[str, Any]]] = []
@@ -233,27 +292,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         }
                     )
                 elif isinstance(content, list):
-                    asst_parts: list[dict[str, Any]] = []
-                    for block in content:
-                        if not isinstance(block, dict):
-                            continue
-                        btype = block.get("type")
-                        if btype == "text":
-                            asst_parts.append({"type": "output_text", "text": block.get("text", "")})
-                        elif btype == "tool_use":
-                            # tool_use becomes a top-level function_call item
-                            input_items.append(
-                                {
-                                    "type": "function_call",
-                                    "call_id": block.get("id", ""),
-                                    "name": block.get("name", ""),
-                                    "arguments": json.dumps(block.get("input", {})),
-                                }
-                            )
-                        elif btype == "thinking":
-                            thinking_text = block.get("thinking", "")
-                            if thinking_text:
-                                asst_parts.append({"type": "output_text", "text": thinking_text})
+                    blocks = tuple(block for block in content if isinstance(block, dict))
+                    input_items.extend(
+                        item
+                        for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key)
+                        if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None
+                    )
+                    asst_parts: list[dict[str, Any]] = [  # mutable-ok: API message payload
+                        {"type": "output_text", "text": block.get("text", "")}  # mutable-ok: API message payload
+                        for block in blocks
+                        if block.get("type") == "text"
+                    ]
                     if asst_parts:
                         input_items.append(
                             {
@@ -514,16 +563,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
 
         for item in response.output:
             if isinstance(item, ResponseReasoningItem):
-                for summary in item.summary:
-                    text = getattr(summary, "text", "")
-                    if text:
-                        content.append(
-                            AnthropicResponseContentBlockThinking(
-                                type="thinking",
-                                thinking=text,
-                                signature=None,
-                            ).model_dump()
-                        )
+                content.extend(self._thinking_blocks_from_reasoning_item(item.summary))
 
             elif isinstance(item, ResponseOutputMessage):
                 for part in item.content:
@@ -555,6 +595,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                             content.append(
                                 AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump()
                             )
+                elif item_type == "reasoning":
+                    content.extend(
+                        self._thinking_blocks_from_reasoning_item(
+                            cast(Iterable[object], item.get("summary") or ()),  # cast-ok: untyped provider json
+                        )
+                    )
                 elif item_type == "function_call":
                     try:
                         input_data = json.loads(item.get("arguments", "{}"))

litellm/llms/anthropic/files/handler.py

@@ -22,19 +22,7 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import CallTypes, LlmProviders, ModelResponse
 
 from ..chat.transformation import AnthropicConfig
-from ..common_utils import AnthropicModelInfo
-
-# Map Anthropic error types to HTTP status codes
-ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = {
-    "invalid_request_error": 400,
-    "authentication_error": 401,
-    "permission_error": 403,
-    "not_found_error": 404,
-    "rate_limit_error": 429,
-    "api_error": 500,
-    "overloaded_error": 503,
-    "timeout_error": 504,
-}
+from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo
 
 
 class AnthropicFilesHandler:

litellm/llms/azure/common_utils.py

@@ -3,6 +3,7 @@ import hashlib
 import json
 import os
 from collections.abc import Callable, Mapping
+from functools import lru_cache
 from typing import Any, Final, Literal, NamedTuple, cast
 
 import httpx
@@ -75,6 +76,24 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
     return {**llm_response_headers, **openai_headers}
 
 
+@lru_cache(maxsize=128)
+def _cached_entra_id_token_provider(
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    scope: str,
+) -> Callable[[], str]:
+    """Build (once per credential set) a bearer token provider backed by a `ClientSecretCredential`.
+
+    The credential caches the access token internally and only talks to Entra ID when it is close
+    to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one
+    per request.
+    """
+    from azure.identity import ClientSecretCredential, get_bearer_token_provider
+
+    return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
+
+
 def get_azure_ad_token_from_entra_id(
     tenant_id: str,
     client_id: str,
@@ -93,8 +112,6 @@ def get_azure_ad_token_from_entra_id(
     Returns:
         callable that returns a bearer token.
     """
-    from azure.identity import ClientSecretCredential, get_bearer_token_provider
-
     verbose_logger.debug("Getting Azure AD Token from Entra ID")
 
     if tenant_id.startswith("os.environ/"):
@@ -120,9 +137,13 @@ def get_azure_ad_token_from_entra_id(
     )
     if _tenant_id is None or _client_id is None or _client_secret is None:
         raise ValueError("tenant_id, client_id, and client_secret must be provided")
-    credential: Final = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
 
-    token_provider: Final = get_bearer_token_provider(credential, scope)
+    token_provider: Final = _cached_entra_id_token_provider(
+        tenant_id=_tenant_id,
+        client_id=_client_id,
+        client_secret=_client_secret,
+        scope=scope,
+    )
 
     verbose_logger.debug("token_provider %s", token_provider)
 

litellm/llms/azure/search/__init__.py

@@ -0,0 +1,3 @@
+from litellm.llms.azure.search.transformation import BingGroundingSearchConfig
+
+__all__ = ("BingGroundingSearchConfig",)

litellm/llms/azure/search/transformation.py

@@ -0,0 +1,442 @@
+"""
+Calls the Microsoft Foundry Responses API with the `bing_grounding` or `web_search`
+tool to search the web (Grounding with Bing Search).
+
+Microsoft docs: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding
+
+Setup:
+    1. Set BING_GROUNDING_PROJECT_ENDPOINT to the Foundry project endpoint, e.g.
+       https://<account>.services.ai.azure.com/api/projects/<project>
+    2. Set BING_GROUNDING_MODEL to a model deployment in that project (e.g. gpt-4.1);
+       it runs the grounded search and its tokens are billed on that deployment
+    3. Optional: set BING_GROUNDING_CONNECTION_ID to a Grounding with Bing Search
+       project connection id to use the `bing_grounding` tool; without it the
+       project's built-in `web_search` tool is used
+    4. Auth: pass api_key (an Azure API key, sent in the api-key header), or set
+       BING_GROUNDING_TOKEN to an Entra bearer token for scope
+       https://ai.azure.com/.default, or configure azure-identity (AZURE_CLIENT_ID /
+       AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, or any
+       DefaultAzureCredential source) and the token is minted automatically
+
+Usage:
+    response = litellm.search(
+        query="latest AI developments",
+        search_provider="bing_grounding",
+        max_results=5,
+    )
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+import httpx
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.search.transformation import (
+    BaseSearchConfig,
+    SearchResponse,
+    SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_DOCS_URL: Final = "https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding"
+
+PROJECT_ENDPOINT_ENV: Final = "BING_GROUNDING_PROJECT_ENDPOINT"
+MODEL_ENV: Final = "BING_GROUNDING_MODEL"
+CONNECTION_ID_ENV: Final = "BING_GROUNDING_CONNECTION_ID"
+TOKEN_ENV: Final = "BING_GROUNDING_TOKEN"
+
+ENTRA_SCOPE: Final = "https://ai.azure.com/.default"
+
+_RESPONSES_PATH: Final = "/openai/v1/responses"
+_SNIPPET_FALLBACK_LENGTH: Final = 300
+_UPSTREAM_ERROR_STATUS: Final = 502
+_RESPONSE_COST_HEADER: Final = "llm_provider-x-litellm-response-cost"
+
+
+class _Annotation(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    url: str | None = None
+    title: str | None = None
+    start_index: int | None = None
+    end_index: int | None = None
+
+
+class _ContentPart(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    text: str = ""
+    annotations: tuple[_Annotation, ...] = ()
+
+
+class _OutputItem(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    content: tuple[_ContentPart, ...] = ()
+
+
+class _ErrorBody(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    message: str | None = None
+
+
+class _IncompleteDetails(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    reason: str | None = None
+
+
+class _ResponsesEnvelope(BaseModel):
+    """A Foundry Responses API body. `output` is required: a body without it is not a
+    Responses API response and must not be reported as a successful empty search.
+
+    A 200 body can still carry `status` `failed` or `incomplete`; those are surfaced as
+    errors rather than reported as a successful empty search."""
+
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    output: tuple[_OutputItem, ...]
+    status: str | None = None
+    error: _ErrorBody | None = None
+    incomplete_details: _IncompleteDetails | None = None
+
+
+class _ErrorEnvelope(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    error: _ErrorBody | None = None
+
+
+def _unwrap_error_detail(error_message: str) -> str:
+    """
+    Surface the human-readable message inside Foundry's error envelope.
+
+    Tool failures nest a second JSON document as a string inside `error.message`
+    (observed live for `bing_grounding` connection errors), so the unwrap runs twice.
+    Falls back to the raw body for anything else.
+    """
+    try:
+        envelope: Final = _ErrorEnvelope.model_validate_json(error_message)
+    except ValidationError:
+        return error_message
+    message: Final = envelope.error.message if envelope.error else None
+    if message is None:
+        return error_message
+    try:
+        nested: Final = _ErrorBody.model_validate_json(message)
+    except ValidationError:
+        return message
+    return nested.message or message
+
+
+def _snippet(text: str, annotation: _Annotation) -> str:
+    """
+    The text a citation supports, not the citation marker itself.
+
+    A url_citation's start/end indices span the inline marker ("([host](url))"),
+    which follows the claim it backs, so the snippet is the marker's own line up
+    to where the marker starts.
+    """
+    start: Final = annotation.start_index
+    marker_start: Final = start if start is not None and 0 <= start <= len(text) else len(text)
+    claim: Final = text[:marker_start].rsplit("\n", 1)[-1].strip()
+    if claim:
+        return claim[-_SNIPPET_FALLBACK_LENGTH:]
+    return text[:_SNIPPET_FALLBACK_LENGTH]
+
+
+def _citation_results(envelope: _ResponsesEnvelope) -> tuple[SearchResult, ...]:
+    """One result per cited URL: first occurrence wins, order preserved as answered."""
+    cited: Final = tuple(
+        SearchResult(
+            title=annotation.title or "",
+            url=annotation.url or "",
+            snippet=_snippet(part.text, annotation),
+            date=None,
+            last_updated=None,
+        )
+        for item in envelope.output
+        if item.type == "message"
+        for part in item.content
+        if part.type == "output_text"
+        for annotation in part.annotations
+        if annotation.type == "url_citation" and annotation.url
+    )
+    first_by_url: Final = MappingProxyType({result.url: result for result in reversed(cited)})
+    return tuple(first_by_url[url] for url in dict.fromkeys(result.url for result in cited))
+
+
+def _valid_max_results(max_results: object) -> int | None:
+    """A positive-int `max_results`, else None. Rejects bools, an `int` subclass, and
+    non-positive values so neither the request-side `count` nor the response-side cap
+    forwards a value the other would silently ignore.
+    """
+    if isinstance(max_results, bool) or not isinstance(max_results, int):
+        return None
+    return max_results if max_results > 0 else None
+
+
+def _requested_max_results(response_kwargs: Mapping[str, object]) -> int | None:
+    """The unified `max_results` cap the caller asked for, if any.
+
+    The built-in web_search tool has no server-side result-count knob, so the cap is
+    enforced here after the fact; connection mode also honors it as a hard ceiling on
+    top of the tool's `count` hint.
+    """
+    optional_params: Final = response_kwargs.get("optional_params")
+    if not isinstance(optional_params, Mapping):
+        return None
+    return _valid_max_results(optional_params.get("max_results"))
+
+
+def _capped(results: tuple[SearchResult, ...], max_results: int | None) -> tuple[SearchResult, ...]:
+    return results[:max_results] if max_results is not None else results
+
+
+class _SearchConfiguration(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    project_connection_id: str
+    count: int | None = None
+
+
+class _BingGroundingParams(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    search_configurations: tuple[_SearchConfiguration, ...]
+
+
+class _BingGroundingTool(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["bing_grounding"] = "bing_grounding"
+    bing_grounding: _BingGroundingParams
+
+
+class _UserLocation(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["approximate"] = "approximate"
+    country: str
+
+
+class _WebSearchTool(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["web_search"] = "web_search"
+    user_location: _UserLocation | None = None
+
+
+class _ResponsesRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model: str
+    input: str
+    tools: tuple[_BingGroundingTool | _WebSearchTool, ...]
+
+
+def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | _WebSearchTool:
+    connection_id: Final = get_secret_str(CONNECTION_ID_ENV)
+    max_results: Final = optional_params.get("max_results")
+    country: Final = optional_params.get("country")
+    if connection_id:
+        configuration: Final = _SearchConfiguration(
+            project_connection_id=connection_id,
+            count=_valid_max_results(max_results),
+        )
+        return _BingGroundingTool(bing_grounding=_BingGroundingParams(search_configurations=(configuration,)))
+    location: Final = _UserLocation(country=country.upper()) if isinstance(country, str) else None
+    return _WebSearchTool(user_location=location)
+
+
+def _default_entra_token_minter() -> str:
+    from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
+
+    return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
+
+
+class BingGroundingSearchConfig(BaseSearchConfig):
+    def __init__(self, entra_token_minter: Callable[[], str] | None = None) -> None:
+        super().__init__()
+        self._entra_token_minter = entra_token_minter
+
+    @staticmethod
+    def ui_friendly_name() -> str:
+        return "Grounding with Bing Search"
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.validate_environment signature
+        api_key: str | None = None,
+        api_base: str | None = None,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.validate_environment signature
+    ) -> dict[str, str]:  # mutable-ok: the http handler passes this straight to httpx as headers
+        """
+        Validate environment and return headers.
+
+        Returns a new dict rather than mutating ``headers``: the http handler calls this
+        a second time after ``litellm/search/main.py`` already did, so it has to be idempotent.
+        """
+        return {  # mutable-ok: httpx requires a plain dict of headers
+            **headers,
+            **self._auth_header(api_key, api_base),
+            "Content-Type": "application/json",
+        }
+
+    def _auth_header(self, api_key: str | None, api_base: str | None) -> Mapping[str, str]:
+        """
+        A caller-supplied ``api_key`` is an Azure API key and rides the ``api-key`` header;
+        an Entra bearer token (``BING_GROUNDING_TOKEN`` or one minted via azure-identity)
+        rides ``Authorization: Bearer``. Foundry rejects the wrong scheme for each.
+        """
+        if api_key:
+            return MappingProxyType({"api-key": api_key})
+        token: Final = self.resolve_server_api_key(
+            caller_api_key=None,
+            caller_api_base=api_base,
+            key_env_vars=(TOKEN_ENV,),
+            base_env_var=PROJECT_ENDPOINT_ENV,
+            default_api_base=None,
+        ) or self._mint_entra_token(api_base)
+        return MappingProxyType({"Authorization": f"Bearer {token}"})
+
+    def _mint_entra_token(self, caller_api_base: str | None) -> str:
+        self._assert_trusted_api_base_for_server_credential(
+            caller_api_base, None, PROJECT_ENDPOINT_ENV, "Azure AD token"
+        )
+        minter: Final = self._entra_token_minter or _default_entra_token_minter
+        try:
+            return minter()
+        except Exception as e:
+            raise ValueError(
+                f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
+                f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "
+                f"AZURE_CLIENT_SECRET / AZURE_TENANT_ID or any DefaultAzureCredential source) "
+                f"for scope {ENTRA_SCOPE}. Underlying error: {e}"
+            ) from e
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        optional_params: dict[str, object],  # mutable-ok: BaseSearchConfig.get_complete_url signature
+        data: dict[str, object] | list[dict[str, object]] | None = None,  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.get_complete_url signature
+    ) -> str:
+        resolved_base: Final = api_base or get_secret_str(PROJECT_ENDPOINT_ENV)
+        if not resolved_base:
+            raise ValueError(
+                f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project "
+                f"endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>."
+            )
+        trimmed: Final = resolved_base.rstrip("/")
+        if trimmed.endswith(_RESPONSES_PATH):
+            return trimmed
+        return f"{trimmed}{_RESPONSES_PATH}"
+
+    def transform_search_request(
+        self,
+        query: str | list[str],  # mutable-ok: BaseSearchConfig.transform_search_request signature
+        optional_params: dict[str, object],  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_request signature
+    ) -> dict[str, object]:  # mutable-ok: the http handler passes this straight to httpx as the JSON body
+        """
+        Transform Search request to the Foundry Responses API format.
+
+        The unified params map as far as the API allows:
+        - max_results -> the bing_grounding search configuration's `count`; the built-in
+          web_search tool has no result-count knob, so that mode instead caps the returned
+          results after the fact (see transform_search_response)
+        - country -> web_search's approximate `user_location` (bing_grounding's `market`
+          wants a full locale like en-US, which a bare country code cannot fill)
+        - search_domain_filter, max_tokens_per_page -> no API equivalent, dropped
+        """
+        model: Final = get_secret_str(MODEL_ENV)
+        if not model:
+            raise ValueError(
+                f"{MODEL_ENV} is not set. Set it to a model deployment in the Foundry project "
+                f"that runs the grounded search, e.g. gpt-4.1."
+            )
+        request: Final = _ResponsesRequest(
+            model=model,
+            input=" ".join(query) if isinstance(query, list) else query,
+            tools=(_search_tool(optional_params),),
+        )
+        return request.model_dump(mode="json", exclude_none=True)
+
+    def transform_search_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_response signature
+    ) -> SearchResponse:
+        try:
+            parsed: Final = _ResponsesEnvelope.model_validate_json(raw_response.content)
+        except ValidationError as e:
+            raise self.get_error_class(
+                error_message=f"response does not match the Foundry Responses API schema: {e}",
+                status_code=raw_response.status_code,
+                headers=dict(raw_response.headers),  # mutable-ok: BaseSearchConfig.get_error_class signature
+            )
+        if parsed.status == "failed":
+            detail: Final = (
+                parsed.error.message if parsed.error and parsed.error.message else "the grounded search failed"
+            )
+            raise self._upstream_error(detail, raw_response)
+        results: Final = _capped(_citation_results(parsed), _requested_max_results(kwargs))
+        if not results and parsed.status == "incomplete":
+            reason: Final = (
+                parsed.incomplete_details.reason
+                if parsed.incomplete_details and parsed.incomplete_details.reason
+                else "unknown reason"
+            )
+            raise self._upstream_error(f"the grounded search was incomplete: {reason}", raw_response)
+        return self._priced(results)
+
+    def _upstream_error(self, detail: str, raw_response: httpx.Response) -> Exception:
+        return self.get_error_class(
+            error_message=detail,
+            status_code=_UPSTREAM_ERROR_STATUS,
+            headers=dict(raw_response.headers),  # mutable-ok: BaseSearchConfig.get_error_class signature
+        )
+
+    def _priced(self, results: tuple[SearchResult, ...]) -> SearchResponse:
+        """web_search mode runs no paid Grounding with Bing transaction, so it must not
+        inherit the connection-mode ``bing_grounding/search`` price; zero its per-query
+        cost while leaving connection mode to the cost map."""
+        response: Final = SearchResponse(
+            results=list(results),  # mutable-ok: SearchResponse.results is list[SearchResult]
+            object="search",
+        )
+        if get_secret_str(CONNECTION_ID_ENV):
+            return response
+        response._hidden_params[
+            "additional_headers"
+        ] = {  # mutable-ok: response_cost_calculator writes into _hidden_params
+            _RESPONSE_COST_HEADER: 0.0
+        }
+        return response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.get_error_class signature
+    ) -> Exception:
+        detail: Final = _unwrap_error_detail(error_message).rstrip(". ")
+        return BaseLLMException(
+            status_code=status_code,
+            message=f"Grounding with Bing Search: {detail}. See {_DOCS_URL} for details.",
+            headers=headers,
+        )

litellm/llms/azure_ai/azure_model_router/transformation.py

@@ -65,15 +65,24 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
 
         Extracts the actual model used from the Azure response (e.g., gpt-5-nano-2025-08-07)
         and returns it with the azure_ai/ prefix for proper display and cost tracking.
+
+        Also stamps that model onto ``_hidden_params`` so downstream consumers (spend logs,
+        response restamping) can read it instead of guessing the route from the model string.
         """
-        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+        from litellm.llms.azure_ai.common_utils import (
+            AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+            AzureFoundryModelInfo,
+        )
+        from litellm.router_utils.add_retry_fallback_headers import (
+            get_hidden_params_dict,
+        )
 
         # Get base model for the parent call (strips routing prefixes for API compatibility)
         base_model: Final[str] = AzureFoundryModelInfo.get_base_model(model)
 
         # Call parent transform_response first - this will extract the actual model
         # from the raw response (e.g., "gpt-5-nano-2025-08-07")
-        model_response = super().transform_response(
+        transformed_response: Final = super().transform_response(
             model=base_model,
             raw_response=raw_response,
             model_response=model_response,
@@ -86,7 +95,15 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
             api_key=api_key,
             json_mode=json_mode,
         )
-        return model_response
+        selected_model: Final = transformed_response.model
+        if selected_model:
+            # Rebuilt rather than mutated in place: ModelResponseBase declares _hidden_params as a
+            # class-level dict, so an in-place write can bleed into unrelated responses.
+            transformed_response._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # ModelResponse exposes no public hidden-params setter  # mutable-ok: ModelResponse requires _hidden_params to be a plain dict
+                **get_hidden_params_dict(transformed_response),
+                AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model,
+            }
+        return transformed_response
 
     def calculate_additional_costs(self, model: str, prompt_tokens: int, completion_tokens: int) -> dict | None:
         """

litellm/llms/azure_ai/chat/transformation.py

@@ -30,7 +30,12 @@ class AzureFoundryErrorStrings(str, enum.Enum):
     SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
 
 
-NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ("thinking_blocks", "provider_specific_fields", "cache_control")
+NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
+    "thinking_blocks",
+    "reasoning_content",
+    "provider_specific_fields",
+    "cache_control",
+)
 
 
 class AzureAIStudioConfig(OpenAIConfig):
@@ -173,7 +178,8 @@ class AzureAIStudioConfig(OpenAIConfig):
         """
         - Azure AI Studio doesn't support content as a list. This handles:
             1. Strips message fields that are not part of the OpenAI chat-completions
-               schema (thinking_blocks, provider_specific_fields, cache_control).
+               schema (thinking_blocks, reasoning_content, provider_specific_fields,
+               cache_control).
                Azure AI Foundry backends set additionalProperties=false and reject
                these with "Extra inputs are not permitted", which breaks multi-turn
                Anthropic-format clients that echo thinking blocks back as history.

litellm/llms/azure_ai/common_utils.py

@@ -1,9 +1,57 @@
+from collections.abc import Mapping
 from typing import Final, Literal
 
 import litellm
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
+from litellm.types.router import GenericLiteLLMParams
+
+AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
+
+
+def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
+    """
+    Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.
+
+    Accepts the same credential set as the `azure` provider: service principal
+    (`tenant_id` / `client_id` / `client_secret`), a pre-fetched `azure_ad_token`, an OIDC
+    federated token, username/password, or `DefaultAzureCredential` / managed identity.
+    """
+    from litellm.llms.azure.common_utils import get_azure_ad_token
+
+    params = GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams()
+
+    return get_azure_ad_token(params)
+
+
+def get_azure_ai_auth_headers(
+    api_key: str | None,
+    litellm_params: Mapping[str, object] | None = None,
+    api_key_header: AzureAIApiKeyHeader = "Authorization",
+    api_key_env_var: str = "AZURE_AI_API_KEY",
+) -> Mapping[str, str]:
+    """
+    Build the auth headers for an Azure AI Foundry route.
+
+    Prefers the API key when one is configured, and otherwise falls back to Entra ID / OAuth,
+    sending the access token as a bearer token.
+    """
+    if api_key:
+        return {api_key_header: f"Bearer {api_key}" if api_key_header == "Authorization" else api_key}
+
+    azure_ad_token = get_azure_ai_entra_token(litellm_params=litellm_params)
+    if azure_ad_token:
+        return {"Authorization": f"Bearer {azure_ad_token}"}
+
+    raise ValueError(
+        f"Missing Azure AI credentials - set an API key (`api_key` or {api_key_env_var}), or Entra ID / OAuth "
+        "credentials (`tenant_id` + `client_id` + `client_secret`, `azure_ad_token`, an OIDC token, or a managed "
+        "identity with `litellm.enable_azure_ad_token_refresh = True`)"
+    )
+
+
+AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model"
 
 
 class AzureFoundryModelInfo(BaseLLMModelInfo):
@@ -37,13 +85,48 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
             return "model_router"
         return "default"
 
+    @staticmethod
+    def get_model_router_selected_model(hidden_params: Mapping[str, object] | None) -> str | None:
+        """The model Azure Model Router actually served, stamped by ``AzureModelRouterConfig``.
+
+        Reading this beats re-deriving the route from a model string: the stamp is set on the
+        code path that was actually taken, so it holds no matter what the caller named the model.
+        """
+        if not hidden_params:
+            return None
+        selected: Final = hidden_params.get(AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY)
+        if isinstance(selected, str) and selected:
+            return selected
+        return None
+
+    @staticmethod
+    def is_model_router_call(
+        model: str | None = None,
+        hidden_params: Mapping[str, object] | None = None,
+    ) -> bool:
+        """Whether a request went down the Azure Model Router route.
+
+        Prefers the response stamp, then the deployment's litellm model path, and only then the
+        caller-supplied name. The last two go through ``get_azure_ai_route`` so the model-router
+        name heuristic lives in exactly one place.
+        """
+        if AzureFoundryModelInfo.get_model_router_selected_model(hidden_params) is not None:
+            return True
+        deployment_model: Final = (
+            hidden_params.get("litellm_model_name") or hidden_params.get("model") if hidden_params is not None else None
+        )
+        return any(
+            isinstance(candidate, str) and AzureFoundryModelInfo.get_azure_ai_route(candidate) == "model_router"
+            for candidate in (deployment_model, model)
+        )
+
     @staticmethod
     def get_api_base(api_base: str | None = None) -> str | None:
         return api_base or litellm.api_base or get_secret_str("AZURE_AI_API_BASE")
 
     @staticmethod
     def get_api_key(api_key: str | None = None) -> str | None:
-        return api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_AI_API_KEY")
+        return api_key or litellm.api_key or get_secret_str("AZURE_AI_API_KEY")
 
     @property
     def api_version(self, api_version: str | None = None) -> str | None:

litellm/llms/azure_ai/image_edit/flux2_transformation.py

@@ -5,7 +5,10 @@ from typing import Any, Final
 from httpx._types import RequestFiles
 
 import litellm
-from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+from litellm.llms.azure_ai.common_utils import (
+    AzureFoundryModelInfo,
+    get_azure_ai_auth_headers,
+)
 from litellm.llms.azure_ai.image_generation.flux_transformation import (
     AzureFoundryFluxImageGenerationConfig,
 )
@@ -71,16 +74,13 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
         """
         Validate Azure AI Foundry environment and set up authentication
         """
-        api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
-        if not api_key:
-            raise ValueError(
-                f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
-            )
-
         headers.update(
             {
-                "Api-Key": api_key,
+                **get_azure_ai_auth_headers(
+                    api_key=AzureFoundryModelInfo.get_api_key(api_key),
+                    litellm_params=litellm_params,
+                    api_key_header="Api-Key",
+                ),
                 "Content-Type": "application/json",
             }
         )

litellm/llms/azure_ai/image_edit/mai_transformation.py

@@ -3,7 +3,10 @@ from typing import TYPE_CHECKING, Any, Final, cast
 import httpx
 from httpx._types import RequestFiles
 
-from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+from litellm.llms.azure_ai.common_utils import (
+    AzureFoundryModelInfo,
+    get_azure_ai_auth_headers,
+)
 from litellm.llms.azure_ai.image_generation.mai_transformation import (
     AzureFoundryMAIImageGenerationConfig,
 )
@@ -91,15 +94,13 @@ class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig):
         litellm_params: dict | None = None,
         api_base: str | None = None,
     ) -> dict:
-        api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
-        if not api_key:
-            raise ValueError(
-                f"Azure AI API key is required for model {model}. "
-                "Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
+        headers.update(
+            get_azure_ai_auth_headers(
+                api_key=AzureFoundryModelInfo.get_api_key(api_key),
+                litellm_params=litellm_params,
+                api_key_header="api-key",
             )
-
-        headers.update({"api-key": api_key})
+        )
         return headers
 
     def get_complete_url(

litellm/llms/azure_ai/image_edit/transformation.py

@@ -3,7 +3,10 @@ from typing import Final
 import httpx
 
 import litellm
-from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+from litellm.llms.azure_ai.common_utils import (
+    AzureFoundryModelInfo,
+    get_azure_ai_auth_headers,
+)
 from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.utils import _add_path_to_api_base
@@ -30,19 +33,14 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
     ) -> dict:
         """
         Validate Azure AI Foundry environment and set up authentication
-        Uses Api-Key header format
+        Uses the Api-Key header format, or an Entra ID / OAuth bearer token when no key is set
         """
-        api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
-        if not api_key:
-            raise ValueError(
-                f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
-            )
-
         headers.update(
-            {
-                "Api-Key": api_key,  # Azure AI Foundry uses Api-Key header format
-            }
+            get_azure_ai_auth_headers(
+                api_key=AzureFoundryModelInfo.get_api_key(api_key),
+                litellm_params=litellm_params,
+                api_key_header="Api-Key",
+            )
         )
         return headers
 

litellm/llms/azure_ai/ocr/document_intelligence/transformation.py

@@ -26,6 +26,7 @@ from litellm.constants import (
 )
 from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin, encode_url_path_segment
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
 from litellm.llms.base_llm.ocr.transformation import (
     OCR_REQUEST_FORMAT_PARAM,
     BaseOCRConfig,
@@ -236,17 +237,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         """
         Validate environment and return headers for Azure Document Intelligence.
 
-        Authentication uses Ocp-Apim-Subscription-Key header.
+        Authentication uses the Ocp-Apim-Subscription-Key header, or an Entra ID / OAuth bearer
+        token when no subscription key is set.
         """
         # Get API key from environment if not provided
         if api_key is None:
             api_key = get_secret_str(AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR)
 
-        if api_key is None:
-            raise ValueError(
-                "Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter"
-            )
-
         # Validate API base/endpoint is provided
         if api_base is None:
             api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
@@ -257,7 +254,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             )
 
         headers = {
-            "Ocp-Apim-Subscription-Key": api_key,
+            **get_azure_ai_auth_headers(
+                api_key=api_key,
+                litellm_params=litellm_params,
+                api_key_header="Ocp-Apim-Subscription-Key",
+                api_key_env_var=AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR,
+            ),
             "Content-Type": "application/json",
             **headers,
         }

litellm/llms/azure_ai/ocr/transformation.py

@@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     async_convert_url_to_base64,
     convert_url_to_base64,
 )
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
 from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
 from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
 from litellm.secret_managers.main import get_secret_str
@@ -47,17 +48,12 @@ class AzureAIOCRConfig(MistralOCRConfig):
         """
         Validate environment and return headers for Azure AI OCR.
 
-        Azure AI uses Bearer token authentication with AZURE_AI_API_KEY.
+        Authenticates with AZURE_AI_API_KEY, or with an Entra ID / OAuth token when no key is set.
         """
         # Get API key from environment if not provided
         if api_key is None:
             api_key = get_secret_str(AZURE_AI_OCR_API_KEY_ENV_VAR)
 
-        if api_key is None:
-            raise ValueError(
-                "Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params"
-            )
-
         # Validate API base is provided
         if api_base is None:
             api_base = get_secret_str("AZURE_AI_API_BASE")
@@ -68,7 +64,7 @@ class AzureAIOCRConfig(MistralOCRConfig):
             )
 
         headers = {
-            "Authorization": f"Bearer {api_key}",
+            **get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
             "Content-Type": "application/json",
             **headers,
         }

litellm/llms/azure_ai/rerank/transformation.py

@@ -2,12 +2,14 @@
 Translate between Cohere's `/rerank` format and Azure AI's `/rerank` format.
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
 from litellm.llms.cohere.rerank.transformation import CohereRerankConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import RerankResponse
@@ -64,15 +66,13 @@ class AzureAIRerankConfig(CohereRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("AZURE_AI_API_KEY") or litellm.azure_key
 
-        if api_key is None:
-            raise ValueError("Azure AI API key is required. Please set 'AZURE_AI_API_KEY' or 'litellm.azure_key'")
-
         default_headers: Final = {
-            "Authorization": f"Bearer {api_key}",
+            **get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
             "accept": "application/json",
             "content-type": "application/json",
         }

litellm/llms/base_llm/rerank/transformation.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
@@ -24,6 +25,7 @@ class BaseRerankConfig(ABC):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         pass
 

litellm/llms/base_llm/videos/transformation.py

@@ -1,9 +1,10 @@
 import types
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any
 
 import httpx
-from httpx._types import RequestFiles
+from httpx._types import FileContent, RequestFiles
 
 from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
@@ -91,6 +92,14 @@ class BaseVideoConfig(ABC):
             raise ValueError("api_base is required")
         return api_base
 
+    def use_multipart_form_data(self) -> bool:
+        """
+        Whether video create requests without files must still be sent as
+        multipart/form-data (the encoding the OpenAI SDK always uses for
+        /videos), instead of falling back to JSON.
+        """
+        return False
+
     @abstractmethod
     def transform_video_create_request(
         self,
@@ -332,14 +341,18 @@ class BaseVideoConfig(ABC):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, Any] | None = None,
         prefetched_source_data: dict[str, Any] | None = None,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         """
-        Transform the video edit request into a URL and JSON data.
+        Transform the video edit request into a URL plus either JSON data or
+        multipart form fields and files.
 
         Returns:
-            Tuple[str, Dict]: (url, data) for the POST request
+            tuple[str, Mapping[str, object], RequestFiles | None]: (url, data,
+            files). When files is None the handler sends data as JSON; otherwise
+            data holds the form fields and files holds the uploaded source video.
         """
         raise NotImplementedError("video edit is not supported for this provider")
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1617,6 +1617,8 @@ class AmazonConverseConfig(BaseConfig):
         }
         if additional_request_params:
             data["additionalModelRequestFields"] = additional_request_params
+            if "thinking" in additional_request_params:
+                data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",)
         if system_content_blocks:
             data["system"] = system_content_blocks
 
@@ -1801,6 +1803,17 @@ class AmazonConverseConfig(BaseConfig):
                 thinking_blocks_list.append(_redacted_block)
         return thinking_blocks_list
 
+    @staticmethod
+    def thinking_tokens_from_additional_fields(additional_fields: object) -> int | None:
+        """Converse omits thinking tokens from its usage block; they only arrive under
+        ``additionalModelResponseFields`` when ``/usage/output_tokens_details`` is requested."""
+        if not isinstance(additional_fields, Mapping):
+            return None
+        usage: Final = additional_fields.get("usage")
+        if not isinstance(usage, Mapping):
+            return None
+        return AnthropicConfig.thinking_tokens_from_usage(usage)
+
     @staticmethod
     def is_converse_usage_shape(usage_object: Mapping[str, object]) -> bool:
         """Converse-family models report camelCase token counts, not Anthropic's snake_case."""
@@ -1842,6 +1855,7 @@ class AmazonConverseConfig(BaseConfig):
         usage: ConverseTokenUsageBlock,
         reasoning_content: str | None = None,
         thinking_ran: bool = False,
+        provider_reasoning_tokens: int | None = None,
     ) -> Usage:
         input_tokens = usage["inputTokens"]
         output_tokens: Final = usage["outputTokens"]
@@ -1862,9 +1876,14 @@ class AmazonConverseConfig(BaseConfig):
             cache_creation_tokens=cache_creation_input_tokens,
             text_tokens=raw_input_tokens,
         )
-        reasoning_tokens: Final = (
+        estimated_reasoning_tokens: Final = (
             token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0
         )
+        reasoning_tokens: Final = (
+            min(max(0, provider_reasoning_tokens), output_tokens)
+            if provider_reasoning_tokens is not None
+            else estimated_reasoning_tokens
+        )
         completion_tokens_details: Final = (
             CompletionTokensDetailsWrapper(
                 reasoning_tokens=reasoning_tokens,
@@ -2272,6 +2291,9 @@ class AmazonConverseConfig(BaseConfig):
             completion_response["usage"],
             reasoning_content=chat_completion_message.get("reasoning_content"),
             thinking_ran=reasoningContentBlocks is not None,
+            provider_reasoning_tokens=self.thinking_tokens_from_additional_fields(
+                completion_response.get("additionalModelResponseFields")
+            ),
         )
 
         ## HANDLE TOOL CALLS

litellm/llms/bedrock/chat/invoke_handler.py

@@ -331,6 +331,7 @@ class AWSEventStreamDecoder:
         self.json_mode = json_mode
         self._current_tool_name: str | None = None
         self._thinking_ran = False
+        self._provider_reasoning_tokens: int | None = None
 
     def check_empty_tool_call_args(self) -> bool:
         """
@@ -559,14 +560,22 @@ class AWSEventStreamDecoder:
                 tool_use = self._handle_converse_stop_event(content_block_index)
             elif "stopReason" in chunk_data:
                 finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
+                self._provider_reasoning_tokens = AmazonConverseConfig.thinking_tokens_from_additional_fields(
+                    chunk_data.get("additionalModelResponseFields")
+                )
             elif "usage" in chunk_data:
                 usage = converse_config.transform_usage(
                     chunk_data.get("usage", {}),
                     thinking_ran=self._thinking_ran,
+                    provider_reasoning_tokens=self._provider_reasoning_tokens,
                 )
             if thinking_blocks:
                 self._thinking_ran = True
 
+            carries_message_content: Final = any(
+                key in chunk_data for key in ("start", "delta", "contentBlockIndex", "stopReason", "trace")
+            )
+
             model_response_provider_specific_fields: Final = {}
             if "trace" in chunk_data:
                 trace: Final = chunk_data.get("trace")
@@ -577,8 +586,8 @@ class AWSEventStreamDecoder:
                         finish_reason=finish_reason,
                         index=0,  # Always 0 - Bedrock never returns multiple choices
                         delta=Delta(
-                            content=text,
-                            role="assistant",
+                            content=text if carries_message_content else None,
+                            role="assistant" if carries_message_content else None,
                             tool_calls=[tool_use] if tool_use else None,
                             provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
                             thinking_blocks=thinking_blocks,

litellm/llms/bedrock/passthrough/transformation.py

@@ -1,4 +1,5 @@
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final, Optional, cast
 
 from httpx import Response
@@ -93,6 +94,9 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
             endpoint_url,
         )
 
+    def get_bedrock_bearer_token(self, litellm_params: Mapping[str, object]) -> str | None:
+        return None
+
     def sign_request(
         self,
         headers: dict,
@@ -109,6 +113,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
             request_data=request_data or {},
             api_base=api_base,
             model=model,
+            api_key=self.get_bedrock_bearer_token(optional_params),
         )
 
     def logging_non_streaming_response(

litellm/llms/bedrock_mantle/common_utils.py

@@ -13,6 +13,7 @@ global state.
 """
 
 import re
+from collections.abc import Mapping
 from typing import Final
 
 from botocore.exceptions import (
@@ -31,30 +32,39 @@ BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1"
 MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE)
 
 
+def resolve_mantle_bearer_token(api_key: str | None) -> str | None:
+    return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+
+
+def resolve_mantle_region(params: Mapping[str, object]) -> str:
+    region: Final = params.get("aws_region_name")
+    if isinstance(region, str) and region:
+        BaseAWSLLM._validate_aws_region_name(region)
+        return region
+    api_base: Final = params.get("api_base")
+    base: Final = (api_base if isinstance(api_base, str) else None) or get_secret_str("BEDROCK_MANTLE_API_BASE")
+    if base:
+        match: Final = MANTLE_HOST_RE.match(base.rstrip("/"))
+        if match:
+            return match.group(1)
+    return (
+        get_secret_str("BEDROCK_MANTLE_REGION")
+        or get_secret_str("AWS_REGION_NAME")
+        or get_secret_str("AWS_REGION")
+        or BEDROCK_MANTLE_DEFAULT_REGION
+    )
+
+
 class BedrockMantleAuthMixin:
     _aws_signer: BaseAWSLLM
 
     @staticmethod
     def _resolve_bearer_token(api_key: str | None) -> str | None:
-        return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+        return resolve_mantle_bearer_token(api_key)
 
     @staticmethod
     def _resolve_region(params: dict) -> str:
-        region: Final = params.get("aws_region_name")
-        if region:
-            BaseAWSLLM._validate_aws_region_name(region)
-            return region
-        base: Final = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
-        if base:
-            match: Final = MANTLE_HOST_RE.match(base.rstrip("/"))
-            if match:
-                return match.group(1)
-        return (
-            get_secret_str("BEDROCK_MANTLE_REGION")
-            or get_secret_str("AWS_REGION_NAME")
-            or get_secret_str("AWS_REGION")
-            or BEDROCK_MANTLE_DEFAULT_REGION
-        )
+        return resolve_mantle_region(params)
 
     def sign_request(
         self,

litellm/llms/bedrock_mantle/passthrough/transformation.py

@@ -0,0 +1,71 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final, Literal, Optional
+
+from httpx import Response
+
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig
+from litellm.llms.bedrock_mantle.common_utils import (
+    MANTLE_HOST_RE,
+    resolve_mantle_bearer_token,
+    resolve_mantle_region,
+)
+from litellm.types.utils import LlmProviders
+
+if TYPE_CHECKING:
+    from litellm.types.utils import CostResponseTypes
+
+
+class BedrockMantlePassthroughConfig(BedrockPassthroughConfig):
+    """Native Bedrock runtime passthrough (InvokeModel, Converse) for deployments declared as bedrock_mantle.
+
+    The Mantle host only serves the OpenAI-compatible surface, so a Mantle api_base lends its region and the
+    request itself goes to bedrock-runtime, signed with the deployment's Bearer token or SigV4 credentials.
+    """
+
+    def _get_aws_region_name(
+        self,
+        optional_params: Mapping[str, object],
+        model: str | None = None,
+        model_id: str | None = None,
+    ) -> str:
+        return resolve_mantle_region(optional_params)
+
+    def get_runtime_endpoint(
+        self,
+        api_base: str | None,
+        aws_bedrock_runtime_endpoint: str | None,
+        aws_region_name: str,
+        endpoint_type: Literal["runtime", "agent", "agentcore"] | None = "runtime",
+    ) -> tuple[str, str]:
+        is_mantle_host: Final = api_base is not None and MANTLE_HOST_RE.match(api_base.rstrip("/")) is not None
+        return super().get_runtime_endpoint(
+            api_base=None if is_mantle_host else api_base,
+            aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+            aws_region_name=aws_region_name,
+            endpoint_type=endpoint_type,
+        )
+
+    def get_bedrock_bearer_token(self, litellm_params: Mapping[str, object]) -> str | None:
+        api_key: Final = litellm_params.get("api_key")
+        return resolve_mantle_bearer_token(api_key if isinstance(api_key, str) else None)
+
+    def logging_non_streaming_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        httpx_response: Response,
+        request_data: dict,  # mutable-ok: mirrors the inherited BedrockPassthroughConfig signature
+        logging_obj: Logging,
+        endpoint: str,
+    ) -> Optional["CostResponseTypes"]:
+        is_converse: Final = "invoke" not in endpoint and "converse" in endpoint
+        shape_provider: Final = LlmProviders.BEDROCK.value if is_converse else custom_llm_provider
+        return super().logging_non_streaming_response(
+            model=model,
+            custom_llm_provider=shape_provider,
+            httpx_response=httpx_response,
+            request_data=request_data,
+            logging_obj=logging_obj,
+            endpoint=endpoint,
+        )

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -15,8 +15,12 @@ role / access key / profile / web identity), signed via the shared
 BaseAWSLLM._sign_request after the request body is finalized.
 """
 
+import json
+from collections.abc import Mapping
 from typing import Any, Final
 
+from typing_extensions import ReadOnly, TypedDict
+
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@@ -50,6 +54,33 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 
 _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
 
+_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
+_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
+_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
+
+
+class _RewrittenOutputTextBlock(TypedDict):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+
+
+class _RewrittenAssistantMessageItem(TypedDict):
+    type: ReadOnly[str]
+    role: ReadOnly[str]
+    content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
+
+
+class _RewrittenCompactionItem(TypedDict):
+    type: ReadOnly[str]
+    encrypted_content: ReadOnly[str]
+
+
+class _RewrittenFunctionCallItem(TypedDict):
+    type: ReadOnly[str]
+    call_id: ReadOnly[str]
+    name: ReadOnly[str]
+    arguments: ReadOnly[str]
+
 
 class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
     def __init__(
@@ -155,6 +186,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         headers: dict,
     ) -> dict:
         remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
+        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
         request_params: Final = (
             {
                 **response_api_optional_request_params,
@@ -168,7 +200,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return super().transform_responses_api_request(
             model=model,
-            input=remaining_input,
+            input=normalized_input,
             response_api_optional_request_params=request_params,
             litellm_params=litellm_params,
             headers=headers,
@@ -210,6 +242,91 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
 
+    @staticmethod
+    def _agent_message_text(item: "Mapping[str, Any]") -> str:
+        content: Final = item.get("content")
+        if not isinstance(content, list):
+            return ""
+        return "".join(
+            str(block.get("text") or block.get("encrypted_content") or "")
+            for block in content
+            if isinstance(block, dict)
+        )
+
+    @classmethod
+    def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
+        text: Final = cls._agent_message_text(item)
+        if not text:
+            return None
+        rewritten: Final[_RewrittenAssistantMessageItem] = {
+            "type": "message",
+            "role": "assistant",
+            "content": ({"type": "output_text", "text": text},),
+        }
+        return rewritten
+
+    @staticmethod
+    def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
+        encrypted_content: Final = item.get("encrypted_content")
+        if not isinstance(encrypted_content, str) or not encrypted_content:
+            return None
+        rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
+        return rewritten
+
+    @staticmethod
+    def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
+        call_id: Final = item.get("call_id")
+        if not isinstance(call_id, str) or not call_id:
+            return None
+        action: Final = item.get("action")
+        rewritten: Final[_RewrittenFunctionCallItem] = {
+            "type": "function_call",
+            "call_id": call_id,
+            "name": "local_shell",
+            "arguments": json.dumps(action) if isinstance(action, dict) else "{}",
+        }
+        return rewritten
+
+    @classmethod
+    def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]":
+        """Returns (normalized item or None to drop it, original type when rewritten)."""
+        if not isinstance(item, dict):
+            return item, None
+        item_type: Final = item.get("type")
+        if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE:
+            return cls._normalize_agent_message_item(item), item_type
+        if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
+            return cls._normalize_context_compaction_item(item), item_type
+        if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
+            return cls._normalize_local_shell_call_item(item), item_type
+        return item, None
+
+    @classmethod
+    def _normalize_codex_input_items(
+        cls,
+        input: "str | ResponseInputParam",
+    ) -> "str | ResponseInputParam":
+        """Rewrite Codex history item types Mantle rejects with 400 "Invalid
+        'input': value did not match any expected variant" into supported
+        equivalents. `agent_message` (Codex multi-agent traffic; its
+        encrypted_content slot carries the plaintext payload when the model
+        never issued encrypted args) becomes an assistant message,
+        `context_compaction` becomes the `compaction` spelling Mantle accepts,
+        and `local_shell_call` becomes the function_call its recorded
+        function_call_output already pairs with.
+        """
+        if not isinstance(input, list):
+            return input
+        normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input)
+        rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))
+        if rewritten_types:
+            verbose_logger.warning(
+                "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
+                rewritten_types,
+            )
+        kept: Final = [item for item, _ in normalized if item is not None]  # mutable-ok: ResponseInputParam is a list
+        return kept  # pyright: ignore[reportReturnType]  # Codex passthrough items sit outside the OpenAI input union
+
     def map_openai_params(
         self,
         response_api_optional_params: ResponsesAPIOptionalRequestParams,

litellm/llms/cerebras/chat.py

@@ -68,6 +68,8 @@ class CerebrasConfig(OpenAIGPTConfig):
             "tool_choice",
             "tools",
             "user",
+            "max_retries",
+            "extra_headers",
         ]
 
         # Only add reasoning_effort for models that support it

litellm/llms/cohere/rerank/transformation.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -81,6 +82,7 @@ class CohereRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("COHERE_API_KEY") or get_secret_str("CO_API_KEY") or litellm.cohere_key

litellm/llms/custom_httpx/http_handler.py

@@ -9,7 +9,7 @@ import threading
 import time
 from collections.abc import AsyncIterable, Callable, Iterable, Mapping
 from http.cookiejar import CookieJar, DefaultCookiePolicy
-from typing import TYPE_CHECKING, Any, Final, Optional, TypeAlias, TypedDict
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, TypeAlias, TypedDict
 
 import certifi
 import httpx
@@ -933,11 +933,83 @@ class AsyncHTTPHandler:
         response.raise_for_status()
         return response
 
+    # Strong references to finalizer-scheduled client-close tasks. A bare
+    # create_task() result may be garbage-collected before it runs, leaving
+    # the underlying aiohttp session unclosed ("Unclosed client session").
+    # Mirrors LiteLLMAiohttpTransport._background_close_tasks.
+    _finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set()  # mutable-ok: strong refs for pending closes
+
+    @classmethod
+    def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
+        cls._finalizer_close_tasks.discard(task)
+        if task.cancelled():
+            return
+        exc: Final = task.exception()
+        if exc is not None:
+            verbose_logger.debug("Error closing client at finalization: %s", exc)
+
+    def _aiohttp_session_bound_elsewhere(self, loop: asyncio.AbstractEventLoop) -> bool:
+        """True when the wrapped aiohttp session is bound to a loop other than
+        ``loop`` — awaiting ``aclose()`` here would touch that loop's internals."""
+        from litellm.llms.custom_httpx.aiohttp_transport import (
+            LiteLLMAiohttpTransport,
+        )
+
+        transport: Final = getattr(self._client, "_transport", None)
+        if not isinstance(transport, LiteLLMAiohttpTransport):
+            return False
+        session: Final = transport.client
+        if not isinstance(session, ClientSession) or session.closed:
+            return False
+        return getattr(session, "_loop", None) is not loop
+
+    def _dispose_wrapped_aiohttp_session(self) -> None:
+        """Dispose the wrapped aiohttp session when ``aclose()`` cannot run here.
+
+        Finalization either has no running loop, or a loop the session is not
+        bound to. Delegating to the transport's lifecycle-aware disposal picks
+        the safe path per session state (async close on its own loop, threadsafe
+        handoff to a loop running elsewhere, or the synchronous connector
+        teardown that flips the flags ``ClientSession.__del__`` checks), so no
+        "Unclosed client session" / "Unclosed connector" warnings fire at
+        garbage collection.
+        """
+        from litellm.llms.custom_httpx.aiohttp_transport import (
+            LiteLLMAiohttpTransport,
+        )
+
+        transport: Final = getattr(self._client, "_transport", None)
+        if not isinstance(transport, LiteLLMAiohttpTransport):
+            return
+        # A shared session (e.g. the proxy's) is never this handler's to close.
+        if not getattr(transport, "_owns_session", False):
+            return
+        session: Final = transport.client
+        if isinstance(session, ClientSession) and not session.closed:
+            transport._close_recycled_session(session)  # pyright: ignore[reportPrivateUsage]  # deliberate reuse of the transport's lifecycle-aware disposal; an async close can never run in this context
+
     def __del__(self) -> None:
         try:
             if not _handler_may_close_client(sys.getrefcount(self._client), self._owns_client):
                 return
-            asyncio.get_running_loop().create_task(self._client.aclose())
+            try:
+                loop: Final = asyncio.get_running_loop()
+            except RuntimeError:
+                # No running loop at finalization time (worker threads after
+                # their loop closed, interpreter/worker shutdown, GC in a
+                # sync context). An async close can never run here.
+                self._dispose_wrapped_aiohttp_session()
+                return
+            if self._aiohttp_session_bound_elsewhere(loop):
+                # GC ran on a live loop (e.g. the app's) but the session
+                # belongs to another, possibly dead, loop — awaiting aclose()
+                # here is the cross-loop path the transport refuses.
+                self._dispose_wrapped_aiohttp_session()
+                return
+            task: Final = loop.create_task(self._client.aclose())
+            cls: Final = type(self)
+            cls._finalizer_close_tasks.add(task)
+            task.add_done_callback(cls._on_finalizer_close_done)
         except Exception:
             pass
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, Type
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import httpx
+from httpx._types import FileContent
 from openai.types.file_deleted import FileDeleted
 
 import litellm
@@ -24,6 +25,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
     validated_max_agentic_loops,
 )
 from litellm.litellm_core_utils.asyncify import run_async_function
+from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
 from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@@ -1108,6 +1110,7 @@ class BaseLLMHTTPHandler:
             headers=headers or {},
             model=model,
             optional_params=optional_rerank_params,
+            litellm_params=litellm_params,
         )
 
         api_base = provider_config.get_complete_url(
@@ -1844,6 +1847,7 @@ class BaseLLMHTTPHandler:
         return provider_config.transform_search_response(
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def async_search(
@@ -1942,6 +1946,7 @@ class BaseLLMHTTPHandler:
         return provider_config.transform_search_response(
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def _async_post_anthropic_messages_with_http_error_retry(
@@ -7050,9 +7055,7 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            # Use JSON when no files, otherwise use form data with files
             if files and len(files) > 0:
-                # Use multipart/form-data when files are present
                 response = sync_httpx_client.post(
                     url=api_base,
                     headers=headers,
@@ -7060,9 +7063,14 @@ class BaseLLMHTTPHandler:
                     files=files,
                     timeout=timeout,
                 )
-
+            elif video_generation_provider_config.use_multipart_form_data():
+                response = sync_httpx_client.post(  # rebind-ok: one of three mutually-exclusive branches
+                    url=api_base,
+                    headers=headers,
+                    files=serialize_multipart_form_fields(data),
+                    timeout=timeout,
+                )
             else:
-                # Use JSON content type for POST requests without files
                 response = sync_httpx_client.post(
                     url=api_base,
                     headers=headers,
@@ -7154,20 +7162,26 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            # Use JSON when no files, otherwise use form data with files
-            if files is None or len(files) == 0:
+            if files and len(files) > 0:
                 response = await async_httpx_client.post(
                     url=api_base,
                     headers=headers,
-                    json=data,
+                    data=data,
+                    files=files,
+                    timeout=timeout,
+                )
+            elif video_generation_provider_config.use_multipart_form_data():
+                response = await async_httpx_client.post(  # rebind-ok: one of three mutually-exclusive branches
+                    url=api_base,
+                    headers=headers,
+                    files=serialize_multipart_form_fields(data),
                     timeout=timeout,
                 )
             else:
                 response = await async_httpx_client.post(
                     url=api_base,
                     headers=headers,
-                    data=data,
-                    files=files,
+                    json=data,
                     timeout=timeout,
                 )
 
@@ -7827,6 +7841,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str,
         litellm_params,
         logging_obj,
+        video_file: FileContent | None = None,
         extra_headers: dict[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
         timeout: float | None = None,
@@ -7838,6 +7853,7 @@ class BaseLLMHTTPHandler:
             return self.async_video_edit_handler(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 video_provider_config=video_provider_config,
                 custom_llm_provider=custom_llm_provider,
                 litellm_params=litellm_params,
@@ -7891,9 +7907,10 @@ class BaseLLMHTTPHandler:
             prefetched_source_data = prefetch_resp.json()
 
         try:
-            url, data = video_provider_config.transform_video_edit_request(
+            url, data, files = video_provider_config.transform_video_edit_request(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 headers=headers,
@@ -7912,11 +7929,10 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            response: Final = sync_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=data,
-                timeout=timeout,
+            response: Final = (
+                sync_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
+                if files
+                else sync_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
             )
             response.raise_for_status()
             return video_provider_config.transform_video_edit_response(
@@ -7936,6 +7952,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str,
         litellm_params,
         logging_obj,
+        video_file: FileContent | None = None,
         extra_headers: dict[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
         timeout: float | None = None,
@@ -7987,9 +8004,10 @@ class BaseLLMHTTPHandler:
             prefetched_source_data = prefetch_resp.json()
 
         try:
-            url, data = video_provider_config.transform_video_edit_request(
+            url, data, files = video_provider_config.transform_video_edit_request(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 headers=headers,
@@ -8008,11 +8026,10 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            response: Final = await async_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=data,
-                timeout=timeout,
+            response: Final = await (
+                async_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
+                if files
+                else async_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
             )
             response.raise_for_status()
             return video_provider_config.transform_video_edit_response(

litellm/llms/dashscope/rerank/transformation.py

@@ -22,6 +22,7 @@ as supported only for gte-rerank-v2 / qwen3-vl-rerank.
 Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -85,6 +86,7 @@ class DashScopeRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("DASHSCOPE_API_KEY")

litellm/llms/databricks/cost_calculator.py

@@ -3,10 +3,31 @@ Helper util for handling databricks-specific cost calculation
 - e.g.: handling 'dbrx-instruct-*'
 """
 
+from types import MappingProxyType
 from typing import Final
 
+from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
 from litellm.types.utils import Usage
-from litellm.utils import get_model_info
+
+_LEGACY_ENDPOINT_NAMES: Final = MappingProxyType(
+    {
+        "dbrx-instruct": "databricks-dbrx-instruct",
+        "meta-llama-3.1-70b-instruct": "databricks-meta-llama-3-1-70b-instruct",
+        "meta-llama-3.1-405b-instruct": "databricks-meta-llama-3-1-405b-instruct",
+        "mixtral-8x7b-instruct-v0.1": "databricks-mixtral-8x7b-instruct",
+        "bge-large-en": "databricks-bge-large-en",
+        "gte-large-en": "databricks-gte-large-en",
+        "llama-2-70b-chat": "databricks-llama-2-70b-chat",
+    }
+)
+
+
+def _registry_key(model: str) -> str:
+    name: Final = model.removeprefix("databricks/")
+    return next(
+        (key for prefix, key in _LEGACY_ENDPOINT_NAMES.items() if name.startswith(prefix)),
+        name,
+    )
 
 
 def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
@@ -20,36 +41,8 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
     Returns:
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
     """
-    base_model = model
-    if model.startswith("databricks/dbrx-instruct") or model.startswith("dbrx-instruct"):
-        base_model = "databricks-dbrx-instruct"
-    elif model.startswith("databricks/meta-llama-3.1-70b-instruct") or model.startswith("meta-llama-3.1-70b-instruct"):
-        base_model = "databricks-meta-llama-3-1-70b-instruct"
-    elif model.startswith("databricks/meta-llama-3.1-405b-instruct") or model.startswith(
-        "meta-llama-3.1-405b-instruct"
-    ):
-        base_model = "databricks-meta-llama-3-1-405b-instruct"
-    elif (
-        model.startswith("databricks/mixtral-8x7b-instruct-v0.1")
-        or model.startswith("mixtral-8x7b-instruct-v0.1")
-        or model.startswith("databricks/mixtral-8x7b-instruct-v0.1")
-        or model.startswith("mixtral-8x7b-instruct-v0.1")
-    ):
-        base_model = "databricks-mixtral-8x7b-instruct"
-    elif model.startswith("databricks/bge-large-en") or model.startswith("bge-large-en"):
-        base_model = "databricks-bge-large-en"
-    elif model.startswith("databricks/gte-large-en") or model.startswith("gte-large-en"):
-        base_model = "databricks-gte-large-en"
-    elif model.startswith("databricks/llama-2-70b-chat") or model.startswith("llama-2-70b-chat"):
-        base_model = "databricks-llama-2-70b-chat"
-    ## GET MODEL INFO
-    model_info: Final = get_model_info(model=base_model, custom_llm_provider="databricks")
-
-    ## CALCULATE INPUT COST
-
-    prompt_cost: Final[float] = usage["prompt_tokens"] * model_info["input_cost_per_token"]
-
-    ## CALCULATE OUTPUT COST
-    completion_cost: Final = usage["completion_tokens"] * model_info["output_cost_per_token"]
-
-    return prompt_cost, completion_cost
+    return generic_cost_per_token(
+        model=_registry_key(model),
+        usage=usage,
+        custom_llm_provider="databricks",
+    )

litellm/llms/deepinfra/rerank/transformation.py

@@ -2,6 +2,7 @@
 Translate between Cohere's `/rerank` format and Deepinfra's `/rerank` format.
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -67,6 +68,7 @@ class DeepinfraRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("DEEPINFRA_API_KEY")

litellm/llms/fireworks_ai/chat/transformation.py

@@ -504,6 +504,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
                 m = cast(dict, message)
                 m.pop("provider_specific_fields", None)
                 m.pop("thinking_blocks", None)
+                m.pop("reasoning_content", None)
 
         return messages
 

litellm/llms/fireworks_ai/rerank/transformation.py

@@ -4,6 +4,7 @@ Fireworks AI Rerank API transformation
 Reference: https://docs.fireworks.ai/inference-api-reference/rerank
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -102,6 +103,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         api_key = self._get_api_key(api_key)
         if api_key is None:

litellm/llms/gemini/videos/transformation.py

@@ -566,6 +566,7 @@ class GeminiVideoConfig(BaseVideoConfig):
         api_base,
         litellm_params,
         headers,
+        video_file=None,
         extra_body=None,
         prefetched_source_data=None,
     ):

litellm/llms/hosted_vllm/chat/transformation.py

@@ -164,12 +164,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
         """
         Support translating:
         - video files from file_id or file_data to video_url
-        - thinking_blocks on assistant messages are removed, and content lists
-          are converted to strings for vLLM compatibility
+        - thinking_blocks and reasoning_content on assistant messages are removed,
+          and content lists are converted to strings for vLLM compatibility
         """
         for message in messages:
             if message["role"] == "assistant":
                 message.pop("thinking_blocks", None)
+                message.pop("reasoning_content", None)
                 existing_content = message.get("content")
                 if isinstance(existing_content, list):
                     text_parts = []

litellm/llms/hosted_vllm/rerank/transformation.py

@@ -2,6 +2,7 @@
 Transformation logic for Hosted VLLM rerank
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -107,6 +108,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key"

litellm/llms/huggingface/rerank/transformation.py

@@ -1,4 +1,5 @@
 import os
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
@@ -123,6 +124,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
         api_base: str | None = None,
     ) -> dict:
         # Get API credentials

litellm/llms/infinity/rerank/transformation.py

@@ -4,6 +4,7 @@ Transformation logic from Cohere's /v1/rerank format to Infinity's  `/v1/rerank`
 Why separate file? Make it easy to see how transformation works
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
@@ -46,6 +47,7 @@ class InfinityRerankConfig(CohereRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("INFINITY_API_KEY") or get_secret_str("INFINITY_API_KEY") or litellm.infinity_key

litellm/llms/jina_ai/rerank/transformation.py

@@ -6,6 +6,7 @@ Why separate file? Make it easy to see how transformation works
 Docs - https://jina.ai/reranker
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 from httpx import URL, Response
@@ -139,6 +140,7 @@ class JinaAIRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             raise ValueError("api_key is required. Set via `api_key` parameter or `JINA_API_KEY` environment variable.")

litellm/llms/nvidia_nim/rerank/transformation.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final, Literal
 
 import httpx
@@ -152,6 +153,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         """
         Validate that the Nvidia NIM API key is present.

litellm/llms/openai/chat/gpt_5_transformation.py

@@ -16,7 +16,7 @@ def _normalize_reasoning_effort_for_chat_completion(
 ) -> str | None:
     """Convert reasoning_effort to the string format expected by OpenAI chat completion API.
 
-    The chat completion API expects a simple string: 'none', 'low', 'medium', 'high', or 'xhigh'.
+    The chat completion API expects an effort string such as 'low' or 'high'.
     Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}.
     """
     if value is None:

litellm/llms/openai/cost_calculation.py

@@ -134,14 +134,7 @@ def cost_per_second(model: str, custom_llm_provider: str | None, duration: float
 
 
 def _video_resolution_to_cost_field_suffix(resolution: str) -> str | None:
-    """
-    Map usage resolution to a safe suffix for ``output_cost_per_second_<suffix>`` keys.
-
-    Note: Currently only ``output_cost_per_second_1080p`` is explicitly declared in
-    ModelInfo (types/utils.py). Other resolution tiers (e.g., 720p, 4k) can be added
-    to model_prices_and_context_window.json but are not exposed via get_model_info()
-    until added to the ModelInfo TypedDict.
-    """
+    """Map usage resolution to a safe suffix for ``output_cost_per_second_<suffix>`` keys."""
     r: Final = resolution.strip().lower()
     if not r:
         return None

litellm/llms/openai/videos/transformation.py

@@ -1,5 +1,7 @@
 import mimetypes
+from collections.abc import Mapping
 from io import BufferedReader, BytesIO
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, cast
 from urllib.parse import quote
 
@@ -101,6 +103,9 @@ class OpenAIVideoConfig(BaseVideoConfig):
 
         return f"{api_base.rstrip('/')}/videos"
 
+    def use_multipart_form_data(self) -> bool:
+        return True
+
     def transform_video_create_request(
         self,
         model: str,
@@ -499,15 +504,26 @@ class OpenAIVideoConfig(BaseVideoConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, object] | None = None,
         prefetched_source_data: dict[str, object] | None = None,
-    ) -> tuple[str, dict]:
-        original_video_id: Final = extract_original_video_id(video_id)
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         url: Final = f"{api_base.rstrip('/')}/edits"
+
+        if video_file is not None:
+            files: Final[RequestFiles] = (self._video_file_tuple(video_file, "video"),)
+            form_data: Final = (
+                MappingProxyType({"prompt": prompt, **extra_body})
+                if extra_body
+                else MappingProxyType({"prompt": prompt})
+            )
+            return url, form_data, files
+
+        original_video_id: Final = extract_original_video_id(video_id)
         data: Final[dict[str, object]] = {"prompt": prompt, "video": {"id": original_video_id}}
         if extra_body:
             data.update(extra_body)
-        return url, data
+        return url, data, None
 
     def transform_video_edit_response(
         self,
@@ -567,21 +583,22 @@ class OpenAIVideoConfig(BaseVideoConfig):
         else:
             files_list.append((field_name, ("input_reference.png", image, image_content_type)))
 
+    def _video_file_tuple(self, video: FileContent, field_name: str) -> tuple[str, FileTypes]:
+        """
+        Build a multipart field tuple for a video upload with proper video MIME
+        type detection: these paths must send video/mp4, not image/* content types.
+        """
+        filename: Final = getattr(video, "name", None) or "input_video.mp4"
+        content_type: Final = self._get_video_content_type(video=video, filename=filename)
+        return (field_name, (filename, video, content_type))
+
     def _add_video_to_files(
         self,
         files_list: list[tuple[str, FileTypes]],
         video: FileContent,
         field_name: str,
     ) -> None:
-        """
-        Add a video to files with proper video MIME type detection.
-
-        This path is used by POST /videos/characters and must send video/mp4,
-        not image/* content types.
-        """
-        filename: Final = getattr(video, "name", None) or "input_video.mp4"
-        content_type: Final = self._get_video_content_type(video=video, filename=filename)
-        files_list.append((field_name, (filename, video, content_type)))
+        files_list.append(self._video_file_tuple(video, field_name))
 
     def _get_video_content_type(self, video: FileContent, filename: str) -> str:
         guessed_content_type, _ = mimetypes.guess_type(filename)

litellm/llms/runwayml/videos/transformation.py

@@ -1,5 +1,6 @@
 from collections.abc import Mapping, Sequence
 from datetime import datetime
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
@@ -33,6 +34,10 @@ else:
     LiteLLMLoggingObj = Any
 
 
+class RunwayMLError(BaseLLMException):
+    pass
+
+
 class _RunwayTaskResponse(TypedDict, total=False):
     id: ReadOnly[str]
     status: ReadOnly[str]
@@ -41,7 +46,8 @@ class _RunwayTaskResponse(TypedDict, total=False):
     output: ReadOnly[Sequence[str] | str]
     failureCode: ReadOnly[str]
     failure: ReadOnly[str]
-    progress: ReadOnly[int]
+    progress: ReadOnly[float]
+    estimatedCost: ReadOnly[Mapping[str, float]]
 
 
 class _VideoObjectData(TypedDict, extra_items=object):
@@ -56,12 +62,54 @@ def _parse_runway_task_response(raw_response: httpx.Response) -> _RunwayTaskResp
     return response_data
 
 
+_USD_PER_CREDIT: Final = 0.01
+
+_RESOLUTION_AREA_TIERS: Final[tuple[tuple[int, str], ...]] = (
+    (600_000, "480p"),
+    (1_500_000, "720p"),
+    (4_000_000, "1080p"),
+)
+
+
+def _ratio_to_resolution(ratio: object) -> str | None:
+    if not isinstance(ratio, str) or ":" not in ratio:
+        return None
+    width_str, _, height_str = ratio.partition(":")
+    if not (width_str.isdigit() and height_str.isdigit()):
+        return None
+    area: Final = int(width_str) * int(height_str)
+    return next((label for threshold, label in _RESOLUTION_AREA_TIERS if area < threshold), "4k")
+
+
+def _duration_seconds(seconds: str | None) -> float | None:
+    if not seconds:
+        return None
+    try:
+        return float(seconds)
+    except ValueError:
+        return None
+
+
+def _estimated_cost_usd(response_data: _RunwayTaskResponse) -> float | None:
+    estimated_cost: Final = response_data.get("estimatedCost")
+    if not isinstance(estimated_cost, Mapping):
+        return None
+    credits: Final = estimated_cost.get("credits")
+    if not isinstance(credits, (int, float)):
+        return None
+    return float(credits) * _USD_PER_CREDIT
+
+
+def _progress_percent(progress: float) -> int:
+    return min(100, max(0, round(float(progress) * 100)))
+
+
 class RunwayMLVideoConfig(BaseVideoConfig):
     """
     Configuration class for RunwayML video generation.
 
     RunwayML uses a task-based API where:
-    1. POST /v1/image_to_video creates a task
+    1. POST /v1/text_to_video, /v1/image_to_video, or /v1/video_to_video creates a task
     2. The task returns immediately with a task ID
     3. Client must poll or wait for task completion
     """
@@ -195,31 +243,36 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         """
         Transform the video creation request for RunwayML API.
 
-        RunwayML expects:
-        {
-            "model": "gen4_turbo",
-            "promptImage": "https://... or data:image/...",
-            "promptText": "description",
-            "ratio": "1280:720",
-            "duration": 5
-        }
+        RunwayML has three generation endpoints discriminated by which input is
+        present, and each request body rejects unknown fields:
+        - /text_to_video: promptText only (rejects promptImage)
+        - /image_to_video: promptImage (+ optional promptText)
+        - /video_to_video: promptVideo or videoUri (rejects promptImage)
         """
-        # Build the request data
+        merged_params: Final = MappingProxyType(
+            {
+                "model": model,
+                "promptText": prompt,
+                **video_create_optional_request_params,
+            }
+        )
+
+        endpoint: Final = self._select_generation_endpoint(merged_params)
+
         request_data: Final[dict[str, object]] = {
-            "model": model,
-            "promptText": prompt,
+            key: value for key, value in merged_params.items() if endpoint == "image_to_video" or key != "promptImage"
         }
 
-        # Add mapped parameters
-        request_data.update(video_create_optional_request_params)
-
-        # RunwayML uses JSON body, no files multipart
         files_list: Final[RequestFiles] = []
 
-        # Append the specific endpoint for video generation
-        full_api_base: Final = f"{api_base}/image_to_video"
+        return request_data, files_list, f"{api_base}/{endpoint}"
 
-        return request_data, files_list, full_api_base
+    def _select_generation_endpoint(self, request_data: Mapping[str, object]) -> str:
+        if request_data.get("promptVideo") is not None or request_data.get("videoUri") is not None:
+            return "video_to_video"
+        if request_data.get("promptImage") is not None:
+            return "image_to_video"
+        return "text_to_video"
 
     def transform_video_create_response(
         self,
@@ -285,13 +338,15 @@ class RunwayMLVideoConfig(BaseVideoConfig):
             video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model)
 
         # Add usage data for cost tracking
-        usage_data: Final = {}
-        if video_obj and hasattr(video_obj, "seconds") and video_obj.seconds:
-            try:
-                usage_data["duration_seconds"] = float(video_obj.seconds)
-            except (ValueError, TypeError):
-                pass
-        video_obj.usage = usage_data
+        video_obj.usage = {
+            key: value
+            for key, value in (
+                ("duration_seconds", _duration_seconds(video_obj.seconds)),
+                ("video_resolution", _ratio_to_resolution(request_data.get("ratio") if request_data else None)),
+                ("provider_reported_cost_usd", _estimated_cost_usd(response_data)),
+            )
+            if value is not None
+        }
 
         return video_obj
 
@@ -581,8 +636,9 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         if "completedAt" in response_data:
             video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt"))
 
-        if "progress" in response_data:
-            video_data["progress"] = response_data["progress"]
+        progress_value: Final = response_data.get("progress")
+        if progress_value is not None:
+            video_data["progress"] = _progress_percent(progress_value)
 
         if "failureCode" in response_data or "failure" in response_data:
             video_data["error"] = {
@@ -616,6 +672,7 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         api_base,
         litellm_params,
         headers,
+        video_file=None,
         extra_body=None,
         prefetched_source_data=None,
     ):
@@ -646,9 +703,7 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         raise NotImplementedError("video extension is not supported for RunwayML")
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
-        from ...base_llm.chat.transformation import BaseLLMException
-
-        raise BaseLLMException(
+        return RunwayMLError(
             status_code=status_code,
             message=error_message,
             headers=headers,

litellm/llms/together_ai/chat.py

@@ -1,58 +0,0 @@
-"""
-Support for OpenAI's `/v1/chat/completions` endpoint.
-
-Calls done in OpenAI/openai.py as TogetherAI is openai-compatible.
-
-Docs: https://docs.together.ai/reference/completions-1
-"""
-
-from typing import Final
-
-from litellm._logging import verbose_logger
-from litellm.utils import supports_function_calling
-
-from ..openai.chat.gpt_transformation import OpenAIGPTConfig
-
-
-class TogetherAIConfig(OpenAIGPTConfig):
-    def get_supported_openai_params(self, model: str) -> list:
-        """
-        Only some together models support response_format / tool calling
-
-        Docs: https://docs.together.ai/docs/json-mode
-        """
-        # Use supports_function_calling() — which reads _get_model_info_helper
-        # directly — instead of get_model_info(). get_model_info() calls
-        # get_supported_openai_params() as its first step, which routes back
-        # into this method for together_ai models, creating a recursion that
-        # only terminates when Python's recursion limit or the "not mapped"
-        # exception in _get_model_info_helper is hit (~332 deep calls).
-        supports_fc: bool | None = None
-        try:
-            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
-        except Exception as e:
-            verbose_logger.debug("Error getting supported openai params: %s", e)
-
-        optional_params: Final = super().get_supported_openai_params(model)
-        if supports_fc is not True:
-            verbose_logger.debug(
-                "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
-            )
-            optional_params.remove("tools")
-            optional_params.remove("tool_choice")
-            optional_params.remove("function_call")
-            optional_params.remove("response_format")
-        return optional_params
-
-    def map_openai_params(
-        self,
-        non_default_params: dict,
-        optional_params: dict,
-        model: str,
-        drop_params: bool,
-    ) -> dict:
-        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
-
-        if "response_format" in mapped_openai_params and mapped_openai_params["response_format"] == {"type": "text"}:
-            mapped_openai_params.pop("response_format")
-        return mapped_openai_params

litellm/llms/together_ai/chat/__init__.py

@@ -0,0 +1,3 @@
+from .transformation import TogetherAIChatConfig as TogetherAIChatConfig
+
+TogetherAIConfig = TogetherAIChatConfig

litellm/llms/together_ai/chat/transformation.py

@@ -0,0 +1,48 @@
+"""
+Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/completions`.
+
+Docs: https://docs.together.ai/docs/chat-overview
+"""
+
+from types import MappingProxyType
+from typing import Final
+
+from litellm._logging import verbose_logger
+from litellm.utils import supports_function_calling
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+FUNCTION_CALLING_ONLY_PARAMS: Final = ("tools", "tool_choice", "function_call", "response_format")
+PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
+
+
+class TogetherAIChatConfig(OpenAIGPTConfig):
+    def get_supported_openai_params(self, model: str) -> list:
+        supports_fc: bool | None = None
+        try:
+            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
+        except Exception as e:
+            verbose_logger.debug("Error getting supported openai params: %s", e)
+
+        supported_params: Final = super().get_supported_openai_params(model)
+        if supports_fc is True:
+            return supported_params
+        verbose_logger.debug(
+            "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
+        )
+        return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
+            param for param in supported_params if param not in FUNCTION_CALLING_ONLY_PARAMS
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+
+        if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
+            mapped_openai_params.pop("response_format")
+        return mapped_openai_params

litellm/llms/together_ai/rerank/handler.py

@@ -16,11 +16,16 @@ from litellm.llms.together_ai.rerank.transformation import TogetherAIRerankConfi
 from litellm.types.rerank import RerankRequest, RerankResponse
 
 
+def _rerank_url(api_base: str) -> str:
+    return f"{api_base.rstrip('/')}/rerank"
+
+
 class TogetherAIRerank(BaseLLM):
     def rerank(
         self,
         model: str,
         api_key: str,
+        api_base: str,
         query: str,
         documents: list[str | dict[str, Any]],
         top_n: int | None = None,
@@ -46,10 +51,10 @@ class TogetherAIRerank(BaseLLM):
             raise ValueError("TogetherAI does not support max_chunks_per_doc")
 
         if _is_async:
-            return self.async_rerank(request_data_dict, api_key)  # Call async method
+            return self.async_rerank(request_data_dict, api_key, api_base)
 
         response: Final = client.post(
-            "https://api.together.xyz/v1/rerank",
+            _rerank_url(api_base),
             headers={
                 "accept": "application/json",
                 "content-type": "application/json",
@@ -69,11 +74,12 @@ class TogetherAIRerank(BaseLLM):
         self,
         request_data_dict: dict[str, Any],
         api_key: str,
+        api_base: str,
     ) -> RerankResponse:
         client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.TOGETHER_AI)  # Use async client
 
         response: Final = await client.post(
-            "https://api.together.xyz/v1/rerank",
+            _rerank_url(api_base),
             headers={
                 "accept": "application/json",
                 "content-type": "application/json",

litellm/llms/vertex_ai/interactions/transformation.py

@@ -0,0 +1,149 @@
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Final
+
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
+from litellm.llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig
+from litellm.llms.vertex_ai.common_utils import validate_vertex_location
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+VERTEX_INTERACTIONS_API_VERSION: Final = "v1beta1"
+VERTEX_INTERACTIONS_DEFAULT_LOCATION: Final = "global"
+
+
+@dataclass(frozen=True, slots=True)
+class VertexInteractionsTarget:
+    base_url: str
+    project_id: str
+    location: str
+
+    @property
+    def collection_url(self) -> str:
+        return (
+            f"{self.base_url}/{VERTEX_INTERACTIONS_API_VERSION}"
+            f"/projects/{self.project_id}/locations/{self.location}/interactions"
+        )
+
+    def interaction_url(self, interaction_id: str) -> str:
+        encoded_interaction_id: Final = encode_url_path_segment(interaction_id, field_name="interaction_id")
+        return f"{self.collection_url}/{encoded_interaction_id}"
+
+
+class VertexAIInteractionsConfig(VertexBase, GoogleAIStudioInteractionsConfig):
+    def __init__(
+        self,
+        mint_access_token: Callable[[VERTEX_CREDENTIALS_TYPES | None, str | None], tuple[str, str]] | None = None,
+    ) -> None:
+        super().__init__()
+        self._mint_access_token: Final[Callable[[VERTEX_CREDENTIALS_TYPES | None, str | None], tuple[str, str]]] = (
+            mint_access_token or self._mint_access_token_with_vertex_base
+        )
+
+    def _mint_access_token_with_vertex_base(
+        self,
+        credentials: VERTEX_CREDENTIALS_TYPES | None,
+        project_id: str | None,
+    ) -> tuple[str, str]:
+        return self._ensure_access_token(
+            credentials=credentials, project_id=project_id, custom_llm_provider="vertex_ai"
+        )
+
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.VERTEX_AI
+
+    @property
+    def api_version(self) -> str:
+        return VERTEX_INTERACTIONS_API_VERSION
+
+    def get_default_vertex_location(self) -> str:
+        return VERTEX_INTERACTIONS_DEFAULT_LOCATION
+
+    def _mint(self, litellm_params: GenericLiteLLMParams) -> tuple[str, str]:
+        raw_params: Final = litellm_params.model_dump()
+        return self._mint_access_token(
+            self.safe_get_vertex_ai_credentials(raw_params),
+            self.safe_get_vertex_ai_project(raw_params),
+        )
+
+    def _target(self, api_base: str | None, litellm_params: GenericLiteLLMParams) -> VertexInteractionsTarget:
+        _, project_id = self._mint(litellm_params)
+        if not project_id:
+            raise ValueError(
+                "Vertex AI project is required. Set vertex_project, litellm.vertex_project, or VERTEXAI_PROJECT"
+            )
+        location: Final = validate_vertex_location(
+            self.explicit_vertex_ai_location(litellm_params.model_dump()) or VERTEX_INTERACTIONS_DEFAULT_LOCATION
+        )
+        return VertexInteractionsTarget(
+            base_url=self.get_api_base(api_base or None, location),
+            project_id=project_id,
+            location=location,
+        )
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        litellm_params: GenericLiteLLMParams | None,
+    ) -> dict:  # mutable-ok: BaseInteractionsAPIConfig declares plain-dict headers
+        access_token, _ = self._mint(litellm_params or GenericLiteLLMParams())
+        return {  # mutable-ok: BaseInteractionsAPIConfig declares plain-dict headers
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {access_token}",
+            **headers,
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        model: str | None,
+        agent: str | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+        stream: bool | None = None,
+    ) -> str:
+        params: Final = (
+            GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams()
+        )
+        collection_url: Final = self._target(api_base, params).collection_url
+        return f"{collection_url}?alt=sse" if stream else collection_url
+
+    def _interaction_by_id_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        url_suffix: str = "",
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        target: Final = self._target(api_base or None, litellm_params)
+        return f"{target.interaction_url(interaction_id)}{url_suffix}", {}  # mutable-ok: same base contract
+
+    def transform_get_interaction_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        return self._interaction_by_id_request(interaction_id, api_base, litellm_params)
+
+    def transform_delete_interaction_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        return self._interaction_by_id_request(interaction_id, api_base, litellm_params)
+
+    def transform_cancel_interaction_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        return self._interaction_by_id_request(interaction_id, api_base, litellm_params, url_suffix=":cancel")

litellm/llms/vertex_ai/rerank/transformation.py

@@ -4,6 +4,7 @@ Translates from Cohere's `/v1/rerank` input format to Vertex AI Discovery Engine
 Why separate file? Make it easy to see how transformation works
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -74,14 +75,15 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         """
         Validate and set up authentication for Vertex AI Discovery Engine API
         """
         # Get credentials and project info from optional_params (which contains vertex_credentials, etc.)
-        litellm_params: Final = optional_params.copy() if optional_params else {}
-        vertex_credentials: Final = self.safe_get_vertex_ai_credentials(litellm_params)
-        vertex_project: Final = self.safe_get_vertex_ai_project(litellm_params)
+        vertex_params: Final = optional_params.copy() if optional_params else {}
+        vertex_credentials: Final = self.safe_get_vertex_ai_credentials(vertex_params)
+        vertex_project: Final = self.safe_get_vertex_ai_project(vertex_params)
 
         # Get access token using the base class method
         access_token, project_id = self._ensure_access_token(

litellm/llms/vertex_ai/videos/transformation.py

@@ -7,11 +7,11 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer
 
 import base64
 import time
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
 
 import httpx
-from httpx._types import RequestFiles
+from httpx._types import FileContent, RequestFiles
 from typing_extensions import ReadOnly
 
 from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
@@ -677,9 +677,10 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, object] | None = None,
         prefetched_source_data: dict[str, Any] | None = None,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         """
         Build a predictLongRunning edit request from the pre-fetched source video.
 
@@ -727,7 +728,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
                 request_data["parameters"] = vertex_params
 
         edit_url: Final = f"{api_base.rstrip('/')}/{model}:predictLongRunning"
-        return edit_url, request_data
+        return edit_url, request_data, None
 
     def transform_video_edit_response(
         self,

litellm/llms/voyage/rerank/transformation.py

@@ -4,6 +4,7 @@ Transformation logic for Voyage AI's /v1/rerank endpoint.
 Docs - https://docs.voyageai.com/docs/reranker
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -137,6 +138,7 @@ class VoyageRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("VOYAGE_API_KEY") or get_secret_str("VOYAGE_AI_API_KEY")

litellm/llms/watsonx/rerank/transformation.py

@@ -5,6 +5,7 @@ Docs - https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank
 """
 
 import uuid
+from collections.abc import Mapping
 from typing import Any, Final, cast
 
 import httpx
@@ -60,6 +61,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         optional_params = optional_params or {}
 

litellm/main.py

@@ -24,6 +24,7 @@ from concurrent import futures
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from copy import deepcopy
 from functools import partial
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
 
 from litellm._logging import _redact_string
@@ -416,7 +417,7 @@ async def acompletion(
     logprobs: bool | None = None,
     top_logprobs: int | None = None,
     deployment_id=None,
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
@@ -602,7 +603,7 @@ async def acompletion(
         _, custom_llm_provider, _, _ = get_llm_provider(
             model=model,
             custom_llm_provider=custom_llm_provider,
-            api_base=base_url,
+            api_base=kwargs.get("api_base") or base_url,
         )
 
     fallbacks = fallbacks or litellm.model_fallbacks
@@ -1811,6 +1812,56 @@ def _complete_fireworks_ai(
     return response
 
 
+def _complete_together_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+    acompletion: Final = ctx.acompletion
+    api_base: Final = ctx.api_base
+    api_key: Final = ctx.api_key
+    client: Final = _dispatch_client_http(ctx)
+    custom_llm_provider: Final = ctx.custom_llm_provider
+    headers: Final = ctx.headers
+    litellm_params: Final = ctx.litellm_params
+    logging: Final = ctx.logging
+    messages: Final = ctx.messages
+    model: Final = ctx.model
+    model_response: Final = ctx.model_response
+    optional_params: Final = ctx.optional_params
+    provider_config: Final = ctx.provider_config
+    shared_session: Final = ctx.shared_session
+    stream: Final = ctx.stream
+    timeout: Final = ctx.timeout
+
+    try:
+        response: Final = base_llm_http_handler.completion(
+            model=model,
+            messages=messages,
+            headers=headers,
+            model_response=model_response,
+            api_key=api_key,
+            api_base=api_base,
+            acompletion=acompletion,
+            logging_obj=logging,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            shared_session=shared_session,
+            timeout=timeout,
+            client=client,
+            custom_llm_provider=custom_llm_provider,
+            encoding=_get_encoding(),
+            stream=stream,
+            provider_config=provider_config,
+        )
+    except Exception as e:
+        logging.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=str(e),
+            additional_args=MappingProxyType({"headers": headers}),
+        )
+        raise
+
+    return response
+
+
 def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
     acompletion: Final = ctx.acompletion
     api_base: Final = ctx.api_base
@@ -4920,7 +4971,7 @@ def completion(
     logit_bias: dict | None = None,
     user: str | None = None,
     # openai v1.0+ new params
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     response_format: dict | type[BaseModel] | None = None,
     seed: int | None = None,
@@ -5600,6 +5651,8 @@ def completion(
         elif custom_llm_provider == "fireworks_ai":
             ## COMPLETION CALL
             response = _complete_fireworks_ai(_dispatch_ctx)
+        elif custom_llm_provider == "together_ai":
+            response = _complete_together_ai(_dispatch_ctx)
         elif custom_llm_provider == "heroku":
             response = _complete_heroku(_dispatch_ctx)
 
@@ -5649,7 +5702,6 @@ def completion(
             or custom_llm_provider == "volcengine"
             or custom_llm_provider == "anyscale"
             or custom_llm_provider == "openai"
-            or custom_llm_provider == "together_ai"
             or custom_llm_provider == "nebius"
             or custom_llm_provider == "wandb"
             or custom_llm_provider == "clarifai"
@@ -5699,14 +5751,6 @@ def completion(
             response = _complete_openrouter(_dispatch_ctx)
         elif custom_llm_provider == "vercel_ai_gateway":
             response = _complete_vercel_ai_gateway(_dispatch_ctx)
-        elif (
-            custom_llm_provider == "together_ai"
-            or ("togethercomputer" in model)
-            or (model in litellm.together_ai_models)
-        ):
-            """
-            Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility
-            """
         elif custom_llm_provider == "palm":
             raise ValueError(
                 "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en"
@@ -6828,6 +6872,8 @@ def embedding(
                 aembedding=aembedding,
             )
         elif custom_llm_provider == "azure_ai":
+            from litellm.llms.azure_ai.common_utils import get_azure_ai_entra_token
+
             api_base = (
                 api_base  # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
                 or litellm.api_base
@@ -6837,8 +6883,8 @@ def embedding(
             api_key = (
                 api_key
                 or litellm.api_key  # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
-                or litellm.openai_key
                 or get_secret_str("AZURE_AI_API_KEY")
+                or get_azure_ai_entra_token(litellm_params=litellm_params_dict)
             )
 
             ## EMBEDDING CALL
@@ -8566,7 +8612,7 @@ def stream_chunk_builder(
         if len(chunks) == 0:
             return None
         ## Route to the text completion logic
-        first_chunk_with_choices: Final = next((c for c in chunks if c["choices"]), None)
+        first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None)
         if first_chunk_with_choices is not None and isinstance(
             first_chunk_with_choices["choices"][0], litellm.utils.TextChoices
         ):  # route to the text completion logic
@@ -8581,7 +8627,7 @@ def stream_chunk_builder(
         simple_content_parts: Final[list[str]] = []
         is_simple_text_stream = True
         for chunk in chunks:
-            if len(chunk["choices"]) == 0:
+            if not chunk.get("choices"):
                 continue
 
             choice = chunk["choices"][0]
@@ -8647,7 +8693,7 @@ def stream_chunk_builder(
         tool_call_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "tool_calls" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["tool_calls"] is not None
         ]
@@ -8661,7 +8707,7 @@ def stream_chunk_builder(
         function_call_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "function_call" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["function_call"] is not None
         ]
@@ -8674,7 +8720,7 @@ def stream_chunk_builder(
         content_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "content" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["content"] is not None
         ]
@@ -8685,7 +8731,7 @@ def stream_chunk_builder(
         thinking_blocks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "thinking_blocks" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
         ]
@@ -8698,7 +8744,7 @@ def stream_chunk_builder(
         reasoning_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "reasoning_content" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["reasoning_content"] is not None
         ]
@@ -8711,7 +8757,7 @@ def stream_chunk_builder(
         annotation_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "annotations" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["annotations"] is not None
         ]
@@ -8728,7 +8774,7 @@ def stream_chunk_builder(
         audio_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "audio" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["audio"] is not None
         ]
@@ -8742,7 +8788,7 @@ def stream_chunk_builder(
         image_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "images" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["images"] is not None
         ]
@@ -8759,7 +8805,7 @@ def stream_chunk_builder(
         provider_specific_chunks: Final = [
             chunk
             for chunk in chunks
-            if len(chunk["choices"]) > 0
+            if chunk.get("choices")
             and "provider_specific_fields" in chunk["choices"][0]["delta"]
             and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
         ]

litellm/passthrough/main.py

@@ -199,7 +199,7 @@ def llm_passthrough_route(
         api_key=api_key,
     )
 
-    litellm_params_dict: Final = get_litellm_params(**kwargs)
+    litellm_params_dict: Final = get_litellm_params(api_key=api_key, api_base=api_base, **kwargs)
 
     if client is None:
         from litellm.llms.custom_httpx.http_handler import (

litellm/proxy/_types.py

@@ -815,6 +815,7 @@ class LiteLLMRoutes(enum.Enum):
         "/team/member_add",
         "/team/member_delete",
         "/team/member_update",
+        "/team/{team_id}/member/{user_id}/reset_spend",
         "/team/permissions_list",
         "/team/permissions_update",
         "/team/daily/activity",
@@ -1287,6 +1288,16 @@ class RegenerateKeyRequest(GenerateKeyRequest):
 class ResetSpendRequest(LiteLLMPydanticObjectBase):
     reset_to: float
 
+    @field_validator("reset_to", mode="before")
+    @classmethod
+    def reject_bool_reset_to(cls, v):
+        # bool is a subclass of int, so pydantic silently coerces True/False into
+        # 1.0/0.0 for a `float` field: a caller who accidentally sends a boolean
+        # would otherwise get an unintended spend reset instead of a 422.
+        if isinstance(v, bool):
+            raise ValueError("reset_to must be a number, not a boolean")  # noqa: TRY004  # pydantic needs ValueError
+        return v
+
 
 class KeyRequest(LiteLLMPydanticObjectBase):
     keys: list[str] | None = None

litellm/proxy/analytics_endpoints/cache_activity.py

@@ -34,10 +34,18 @@ class CacheActivityFilterOptions(BaseModel):
     models: list[str]
 
 
+class CacheActivityErrorBucket(BaseModel):
+    call_type: str
+    error_code: str
+    error_class: str
+    count: int
+
+
 class CacheActivityResponse(BaseModel):
     groups: list[CacheActivityGroup]
     totals: CacheActivityTotals
     filter_options: CacheActivityFilterOptions
+    error_breakdown: tuple[CacheActivityErrorBucket, ...]
 
 
 GROUPS_SQL: Final = """
@@ -65,6 +73,26 @@ GROUPS_SQL: Final = """
     ORDER BY (COUNT(*)) DESC
 """
 
+ERROR_BREAKDOWN_SQL: Final = """
+    SELECT
+        CASE WHEN sl."call_type" = '' THEN 'Unknown' ELSE sl."call_type" END AS call_type,
+        COALESCE(NULLIF(sl."metadata"->'error_information'->>'error_code', ''), 'Unknown') AS error_code,
+        COALESCE(NULLIF(sl."metadata"->'error_information'->>'error_class', ''), 'Unknown') AS error_class,
+        COUNT(*)::int AS count
+    FROM "LiteLLM_SpendLogs" sl
+    LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token"
+    WHERE
+        sl."status" = 'failure'
+        AND sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
+        AND sl."startTime" <  (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
+        AND ($3::jsonb = '[]'::jsonb
+            OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb)))
+        AND ($4::jsonb = '[]'::jsonb
+            OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb)))
+    GROUP BY 1, 2, 3
+    ORDER BY (COUNT(*)) DESC
+"""
+
 KEY_ALIAS_OPTIONS_SQL: Final = """
     SELECT DISTINCT COALESCE(vt."key_alias", 'Unnamed Key') AS key_alias
     FROM "LiteLLM_SpendLogs" sl
@@ -95,6 +123,7 @@ class _ModelRow(BaseModel):
 
 
 _groups_adapter: Final = TypeAdapter(list[CacheActivityGroup])
+_error_buckets_adapter: Final = TypeAdapter(tuple[CacheActivityErrorBucket, ...])
 _key_alias_rows_adapter: Final = TypeAdapter(list[_KeyAliasRow])
 _model_rows_adapter: Final = TypeAdapter(list[_ModelRow])
 
@@ -120,10 +149,11 @@ async def get_cache_activity(
     key_aliases: Sequence[str],
     models: Sequence[str],
 ) -> CacheActivityResponse:
-    group_rows, key_alias_rows, model_rows = await asyncio.gather(
-        prisma_client.db.query_raw(
-            GROUPS_SQL, start_date, end_date, json.dumps(list(key_aliases)), json.dumps(list(models))
-        ),
+    key_aliases_json: Final = json.dumps(list(key_aliases))
+    models_json: Final = json.dumps(list(models))
+    group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather(
+        prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json),
+        prisma_client.db.query_raw(ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json),
         prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date),
         prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date),
     )
@@ -135,4 +165,5 @@ async def get_cache_activity(
             key_aliases=[row.key_alias for row in _key_alias_rows_adapter.validate_python(key_alias_rows or [])],
             models=[row.model for row in _model_rows_adapter.validate_python(model_rows or [])],
         ),
+        error_breakdown=_error_buckets_adapter.validate_python(error_rows or []),
     )

litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py

@@ -6,21 +6,21 @@ Plugins are stored as metadata + git source references in LiteLLM database.
 Actual plugin files are hosted on GitHub/GitLab/Bitbucket.
 
 Endpoints:
-/claude-code/marketplace.json  - GET  - List plugins for Claude Code discovery
-/claude-code/plugins           - POST - Register a new plugin (create-only)
-/claude-code/plugins           - GET  - List plugins (admin)
-/claude-code/plugins/{name}    - GET  - Get plugin details
-/claude-code/plugins/{name}    - PUT  - Update an existing plugin
-/claude-code/plugins/{name}/enable  - POST - Enable a plugin
-/claude-code/plugins/{name}/disable - POST - Disable a plugin
-/claude-code/plugins/{name}    - DELETE - Delete a plugin
+/claude-code/marketplace.json  - GET  - List plugins for Claude Code discovery (unauthenticated)
+/claude-code/plugins           - POST - Register a new plugin (create-only, proxy admin only)
+/claude-code/plugins           - GET  - List plugins (any authenticated key)
+/claude-code/plugins/{name}    - GET  - Get plugin details (any authenticated key)
+/claude-code/plugins/{name}    - PUT  - Update an existing plugin (proxy admin only)
+/claude-code/plugins/{name}/enable  - POST - Enable a plugin (proxy admin only)
+/claude-code/plugins/{name}/disable - POST - Disable a plugin (proxy admin only)
+/claude-code/plugins/{name}    - DELETE - Delete a plugin (proxy admin only)
 """
 
 import json
 import re
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import Final, Protocol, TypedDict
+from typing import Annotated, Final, Protocol, TypedDict
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
@@ -28,6 +28,7 @@ from fastapi.responses import JSONResponse
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
 from litellm.repositories.table_repositories import ClaudeCodePluginRepository
 from litellm.types.proxy.claude_code_endpoints import (
     ListPluginsResponse,
@@ -221,6 +222,18 @@ def _name_conflict_error(name: str) -> HTTPException:
     )
 
 
+def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None:
+    """Catalog mutations are restricted to proxy admins: marketplace.json is served
+    unauthenticated and any registered/updated entry is immediately installable by
+    every user, so a non-admin key must never be able to add or overwrite one.
+    """
+    if not is_proxy_admin(user_api_key_dict):
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins may modify the Claude Code plugin marketplace."},
+        )
+
+
 @router.post(
     "/claude-code/plugins",
     tags=["Claude Code Marketplace"],
@@ -242,6 +255,8 @@ async def register_plugin(
     the same name already exists it returns 409 Conflict; use
     PUT /claude-code/plugins/{plugin_name} to update an existing plugin.
 
+    Requires a proxy admin API key.
+
     Parameters:
         - name: Plugin name (kebab-case)
         - source: Git source reference (github, url, or git-subdir format)
@@ -271,6 +286,8 @@ async def register_plugin(
     from prisma.errors import UniqueViolationError
 
     try:
+        _require_proxy_admin(user_api_key_dict)
+
         prisma_client: Final = await _get_prisma_client()
 
         if not re.match(r"^[a-z0-9-]+$", request.name):
@@ -468,6 +485,7 @@ async def get_plugin(
 async def update_plugin(
     plugin_name: str,
     request: UpdatePluginRequest,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ):
     """
     Update an existing plugin in the LiteLLM marketplace.
@@ -481,6 +499,8 @@ async def update_plugin(
     Returns 404 if no plugin with the given name exists; use
     POST /claude-code/plugins to create a new plugin.
 
+    Requires a proxy admin API key.
+
     Parameters:
         - plugin_name: Name of the plugin to update (path parameter)
         - source: Git source reference (github, url, or git-subdir format)
@@ -509,6 +529,8 @@ async def update_plugin(
     from prisma.errors import PrismaError
 
     try:
+        _require_proxy_admin(user_api_key_dict)
+
         prisma_client: Final = await _get_prisma_client()
 
         _validate_plugin_source(request.source)
@@ -566,10 +588,14 @@ async def enable_plugin(
     """
     Enable a disabled plugin.
 
+    Requires a proxy admin API key.
+
     Parameters:
         - plugin_name: The name of the plugin to enable
     """
     try:
+        _require_proxy_admin(user_api_key_dict)
+
         prisma_client: Final = await _get_prisma_client()
 
         plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique(
@@ -611,10 +637,14 @@ async def disable_plugin(
     """
     Disable a plugin without deleting it.
 
+    Requires a proxy admin API key.
+
     Parameters:
         - plugin_name: The name of the plugin to disable
     """
     try:
+        _require_proxy_admin(user_api_key_dict)
+
         prisma_client: Final = await _get_prisma_client()
 
         plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique(
@@ -656,10 +686,14 @@ async def delete_plugin(
     """
     Delete a plugin from the marketplace.
 
+    Requires a proxy admin API key.
+
     Parameters:
         - plugin_name: The name of the plugin to delete
     """
     try:
+        _require_proxy_admin(user_api_key_dict)
+
         prisma_client: Final = await _get_prisma_client()
 
         plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique(