Merge remote-tracking branch 'upstream/litellm_internal_staging' into litellm_anthropic_wif_oidc_28607

# Conflicts:
#	basedpyright-code-budget.json
#	litellm/proxy/credential_endpoints/endpoints.py
#	litellm/proxy/management_endpoints/model_management_endpoints.py
#	tests/test_litellm/router_utils/test_fallback_event_handlers.py
#	type-discipline-budget.json

.github/CODEOWNERS

@@ -1,6 +1,9 @@
 /ui/ @yuneng-berri @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
+/ui/Dockerfile
+/ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
 /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri

.github/pull_request_template.md

@@ -1,7 +1,10 @@
+<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
+     everyday engineering language, extremely parsable and readable at a glance. This goes double for
+     the TLDR, User Flow, and Caveats sections -->
+
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
-     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
 
 Problem this solves:
 
@@ -110,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 ## Caveats (if any)
 
-<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
+     short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Include only the tiers that have caveats; drop the empty ones
+     - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
+       it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
+       rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
+       operator must plan around it before rollout
+     - High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
+       unsafe to ship as is
+     - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
+     - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
+     Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
+     human reader
      Leave this section empty if there are none -->
 
 ## QA runbook
@@ -134,6 +149,6 @@ Example checklists:
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
 -->
 
-### Final Attestation
+## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

.github/scripts/e2e_egress_sentinel.py

@@ -0,0 +1,198 @@
+"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
+
+`serve` pins each provider host (`--host`) to a local sink address in the hosts
+file and binds a counting listener on that address, so any connection the proxy
+or the record/replay edge opens to a real provider is redirected to the sink,
+recorded as one line in `--hits-file`, and never leaves the box. The record and
+replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
+pinned provider names), so in a clean replay the sink sees nothing; a single hit
+means a provider call escaped the bundle. `assert-empty` turns that hit file into
+the pass/fail check.
+
+Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
+editing the hosts file both need root); `--sink-address`, `--port`, and
+`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
+a high port under test.
+"""
+
+# ruff: noqa: T201  # CLI script: its stdout/stderr progress and results are the interface
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import Final
+
+_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
+_BLOCK_END: Final = "# END e2e-egress-sentinel"
+
+
+@dataclass(frozen=True, slots=True)
+class ServeConfig:
+    hosts: tuple[str, ...]
+    sink_address: str
+    ports: tuple[int, ...]
+    hits_file: Path
+    hosts_file: Path
+    ready_file: Path | None
+    pid_file: Path | None
+
+
+def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
+    lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
+    return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
+
+
+def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
+    original = hosts_file.read_bytes() if hosts_file.exists() else b""
+    hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
+    return original
+
+
+def _restore_pins(hosts_file: Path, original: bytes) -> None:
+    hosts_file.write_bytes(original)
+
+
+def _bind(sink_address: str, port: int) -> socket.socket:
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    listener.bind((sink_address, port))
+    listener.listen(128)
+    return listener
+
+
+@dataclass(frozen=True, slots=True)
+class _HitLog:
+    path: Path
+    _lock: threading.Lock
+
+    def record(self, *, port: int, peer: tuple[str, int]) -> None:
+        entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
+        with self._lock:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(entry + "\n")
+
+
+def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
+    while not stop.is_set():
+        try:
+            conn, peer = listener.accept()
+        except OSError:
+            return
+        hits.record(port=port, peer=(peer[0], peer[1]))
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def serve(config: ServeConfig) -> int:
+    config.hits_file.write_text("", encoding="utf-8")
+    original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
+    try:
+        listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
+    except OSError as exc:
+        _restore_pins(config.hosts_file, original_hosts)
+        print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
+        return 1
+
+    stop = threading.Event()
+    hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
+    threads = tuple(
+        threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
+        for listener, port in zip(listeners, config.ports)
+    )
+    for thread in threads:
+        thread.start()
+
+    def _handle(_signum: int, _frame: FrameType | None) -> None:
+        stop.set()
+        for listener in listeners:
+            try:
+                listener.close()
+            except OSError:
+                pass
+
+    signal.signal(signal.SIGTERM, _handle)
+    signal.signal(signal.SIGINT, _handle)
+
+    if config.pid_file is not None:
+        config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    if config.ready_file is not None:
+        config.ready_file.write_text("ready", encoding="utf-8")
+    print(
+        f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
+        f"on port(s) {', '.join(str(p) for p in config.ports)}",
+        flush=True,
+    )
+
+    stop.wait()
+    _restore_pins(config.hosts_file, original_hosts)
+    if config.ready_file is not None and config.ready_file.exists():
+        config.ready_file.unlink()
+    if config.pid_file is not None and config.pid_file.exists():
+        config.pid_file.unlink()
+    return 0
+
+
+def assert_empty(hits_file: Path) -> int:
+    if not hits_file.exists():
+        print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
+        return 0
+    hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not hits:
+        print("egress sentinel recorded no provider calls: zero egress")
+        return 0
+    print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
+    for line in hits:
+        print(f"  {line}", file=sys.stderr)
+    return 1
+
+
+def _serve_from_args(args: argparse.Namespace) -> int:
+    config = ServeConfig(
+        hosts=tuple(args.host),
+        sink_address=args.sink_address,
+        ports=tuple(args.port),
+        hits_file=Path(args.hits_file),
+        hosts_file=Path(args.hosts_file),
+        ready_file=Path(args.ready_file) if args.ready_file else None,
+        pid_file=Path(args.pid_file) if args.pid_file else None,
+    )
+    return serve(config)
+
+
+def main(argv: tuple[str, ...]) -> int:
+    parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
+    serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
+    serve_parser.add_argument("--sink-address", default="127.0.0.1")
+    serve_parser.add_argument("--port", action="append", type=int, default=None)
+    serve_parser.add_argument("--hits-file", required=True)
+    serve_parser.add_argument("--hosts-file", default="/etc/hosts")
+    serve_parser.add_argument("--ready-file", default=None)
+    serve_parser.add_argument("--pid-file", default=None)
+
+    assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
+    assert_parser.add_argument("--hits-file", required=True)
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        if args.port is None:
+            args.port = [443]
+        return _serve_from_args(args)
+    return assert_empty(Path(args.hits_file))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(tuple(sys.argv[1:])))

.github/scripts/e2e_fetch_fixture_bundle.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
+ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
+BASE_BRANCH="${3:?base branch required}"
+DEST_DIR="${4:?destination bundle dir required}"
+
+: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
+
+SELECTED="$(
+  gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
+    --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
+    | jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
+)"
+
+if [[ -z "${SELECTED}" ]]; then
+  echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
+  exit 1
+fi
+
+RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
+RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
+ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
+GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
+CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
+
+echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
+
+gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
+
+TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
+if [[ -z "${TARBALL}" ]]; then
+  echo "downloaded artifact contained no tarball" >&2
+  exit 1
+fi
+SIDECAR="${TARBALL}.sha256"
+if [[ ! -f "${SIDECAR}" ]]; then
+  echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
+  exit 1
+fi
+
+echo "verifying bundle against its recorded sha256 digest"
+( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
+
+mkdir -p "${DEST_DIR}"
+tar xzf "${TARBALL}" -C "${DEST_DIR}"
+
+echo "extracted bundle into ${DEST_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

.github/scripts/e2e_pack_fixture_bundle.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <bundle-dir> <out-tarball>" >&2
+  exit 2
+fi
+
+BUNDLE_DIR="$1"
+OUT_TARBALL="$2"
+
+MANIFEST="${BUNDLE_DIR}/manifest.json"
+if [[ ! -f "${MANIFEST}" ]]; then
+  echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
+  exit 1
+fi
+
+echo "packing fixture bundle from ${BUNDLE_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
+
+TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
+if [[ "${TEST_DIRS}" -eq 0 ]]; then
+  echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
+  exit 1
+fi
+echo "  ${TEST_DIRS} recorded test director(ies)"
+
+mkdir -p "$(dirname "${OUT_TARBALL}")"
+tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
+
+OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
+OUT_BASE="$(basename "${OUT_TARBALL}")"
+( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
+
+echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
+cat "${OUT_DIR}/${OUT_BASE}.sha256"

.github/workflows/e2e_record_replay.yml

@@ -0,0 +1,237 @@
+name: "E2E Record and Replay"
+
+on:
+  schedule:
+    - cron: "0 8 * * 6"
+    - cron: "0 8 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
+        type: choice
+        options:
+          - record
+          - replay
+        default: record
+
+permissions:
+  contents: read
+
+jobs:
+  record:
+    name: "Record the e2e suite against real providers"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 6' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Record the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: record
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Pack the fixture bundle
+        run: |
+          .github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
+
+      - name: Publish the fixture bundle
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-fixtures-bundle
+          path: |
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log
+
+  replay:
+    name: "Replay the e2e suite from the pinned bundle with zero egress"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 1-5' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: read
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      GH_TOKEN: ${{ github.token }}
+      OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
+      ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Fetch the pinned fixture bundle by digest
+        env:
+          BASE_BRANCH: ${{ github.ref_name }}
+        run: |
+          .github/scripts/e2e_fetch_fixture_bundle.sh \
+            "${GITHUB_REPOSITORY}" \
+            e2e-fixtures-bundle \
+            "${BASE_BRANCH}" \
+            tests/e2e/.fixtures
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Start the egress sentinel
+        run: |
+          # shellcheck disable=SC2024  # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
+          sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
+            --host api.openai.com \
+            --host api.anthropic.com \
+            --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
+            --ready-file "${RUNNER_TEMP}/egress-ready" \
+            --pid-file "${RUNNER_TEMP}/egress.pid" \
+            > "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
+              cat "${RUNNER_TEMP}/egress-sentinel.log"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "egress sentinel never became ready"
+          cat "${RUNNER_TEMP}/egress-sentinel.log"
+          exit 1
+
+      - name: Replay the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: replay
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Stop the egress sentinel and assert zero provider egress
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
+            sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
+            sleep 2
+          fi
+          python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/test-code-quality.yml

@@ -131,6 +131,9 @@ jobs:
       - name: check_e2e_no_raw_requests
         run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
 
+      - name: check_migrations_no_data_rewrites
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-unit.yml

@@ -164,6 +164,7 @@ jobs:
               tests/test_litellm/proxy/public_endpoints
               tests/test_litellm/proxy/prompts
               tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/rerank_endpoints
               tests/test_litellm/proxy/realtime_endpoints
               tests/test_litellm/proxy/ui_crud_endpoints
               tests/test_litellm/proxy/config_resolvers

CLAUDE.md

@@ -37,13 +37,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
 
 Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
-If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
+If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
-- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
+- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 
@@ -79,6 +80,8 @@ Do not put names of customers or customer company names in code, PR descriptions
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
+
 Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
 
 - Composition over inheritance

basedpyright-code-budget.json

@@ -1,18 +1,18 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 18505
   },
   "reportArgumentType": {
-    "limit": 2566
+    "limit": 2564
   },
   "reportAssignmentType": {
     "limit": 320
   },
   "reportAttributeAccessIssue": {
-    "limit": 488
+    "limit": 483
   },
   "reportCallIssue": {
-    "limit": 114
+    "limit": 113
   },
   "reportConstantRedefinition": {
     "limit": 40
@@ -24,7 +24,7 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 6049
+    "limit": 5976
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -45,7 +45,7 @@
     "limit": 35
   },
   "reportInvalidTypeForm": {
-    "limit": 35
+    "limit": 34
   },
   "reportInvalidTypeVarUse": {
     "limit": 2
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5659
   },
   "reportMissingTypeArgument": {
-    "limit": 15553
+    "limit": 15504
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1061
+    "limit": 1058
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1822
+    "limit": 1810
   },
   "reportRedeclaration": {
     "limit": 8
@@ -99,31 +99,31 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44653
+    "limit": 44530
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39003
+    "limit": 38828
   },
   "reportUnknownParameterType": {
-    "limit": 19883
+    "limit": 19847
   },
   "reportUnknownVariableType": {
-    "limit": 30557
+    "limit": 30386
   },
   "reportUnnecessaryCast": {
     "limit": 117
   },
   "reportUnnecessaryComparison": {
-    "limit": 699
+    "limit": 697
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 836
+    "limit": 833
   },
   "reportUntypedBaseClass": {
     "limit": 0

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -14,6 +14,8 @@ from litellm.constants import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.integrations.prometheus import PrometheusLogger
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -351,7 +353,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""

litellm/__init__.py

@@ -199,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
 )
 log_raw_request_response: bool = False
+log_client_error_tracebacks: bool = False
 request_correlation_in_logs: bool = False
 redact_messages_in_exceptions: Optional[bool] = False
 redact_user_api_key_info: Optional[bool] = False
@@ -1628,6 +1629,9 @@ if TYPE_CHECKING:
         AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -1801,6 +1805,9 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,

litellm/_lazy_imports_registry.py

@@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
+    "TogetherAIChatConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
     "GoogleAIStudioGeminiConfig",
@@ -242,6 +243,7 @@ LLM_CONFIG_NAMES: Final = (
     "OpenRouterResponsesAPIConfig",
     "BedrockMantleResponsesAPIConfig",
     "GoogleAIStudioInteractionsConfig",
+    "VertexAIInteractionsConfig",
     "OpenAIOSeriesConfig",
     "AnthropicSkillsConfig",
     "BaseSkillsAPIConfig",
@@ -740,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "AmazonMantleMessagesConfig",
     ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
+    "TogetherAIChatConfig": (
+        ".llms.together_ai.chat.transformation",
+        "TogetherAIChatConfig",
+    ),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (
         ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",
@@ -977,6 +983,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.gemini.interactions.transformation",
         "GoogleAIStudioInteractionsConfig",
     ),
+    "VertexAIInteractionsConfig": (
+        ".llms.vertex_ai.interactions.transformation",
+        "VertexAIInteractionsConfig",
+    ),
     "OpenAIOSeriesConfig": (
         ".llms.openai.chat.o_series_transformation",
         "OpenAIOSeriesConfig",

litellm/caching/redis_cluster_node_isolation.py

@@ -18,6 +18,14 @@ already does when one of its pooled connections errors), leaving every other nod
 connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
+
+redis-py 8.x fixed this upstream with gentler machinery than this override's
+``node.disconnect()`` (which also kills connections other coroutines are mid-operation
+on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
+killed connection): it marks in-use connections for reconnect only after their current
+operation completes, disconnects only the idle pooled ones, and defers reinitialization
+to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
+recovery API, the factory returns the base ``RedisCluster`` unmodified.
 """
 
 import asyncio
@@ -72,8 +80,16 @@ class _ClusterAttrs(Protocol):
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
 
 
-def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
-    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+def get_litellm_async_redis_cluster_class(
+    cluster_node_class: type | None = None,
+) -> type["_AsyncRedisClusterType"]:
+    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
+    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch
+    tears down the whole cluster client.
+
+    ``cluster_node_class`` exists for dependency injection in tests; production callers
+    leave it unset and the installed ``ClusterNode`` is used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -81,7 +97,10 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     """
     import redis
     from redis.asyncio.cluster import (
-        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+        ClusterNode as _AsyncClusterNode,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # same stale-stub gap as the import above
     )
     from redis.cluster import get_node_name
     from redis.commands import READ_COMMANDS
@@ -98,6 +117,15 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     from redis.exceptions import ConnectionError as _RedisConnectionError
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
+    node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    if hasattr(node_class, "update_active_connections_for_reconnect"):
+        verbose_logger.debug(
+            "redis-py %s recovers a node-level connection error per-connection upstream; "
+            "using the base RedisCluster without litellm's node-isolation override.",
+            redis.__version__,
+        )
+        return _BaseAsyncRedisCluster
+
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(
             "redis-py %s is not in the set this cluster-teardown-storm fix was verified "

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -35,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
 )
 from litellm.responses.utils import normalize_responses_api_stream_options
 from litellm.types.llms.openai import (
+    REASONING_EFFORT,
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
     ChatCompletionToolCallChunk,
@@ -1113,22 +1114,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
         )
 
-        # If string is passed, map with optional summary based on flag/env var
-        if reasoning_effort == "none":
-            return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
-        elif reasoning_effort == "high":
-            return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
-        elif reasoning_effort == "xhigh":
-            return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
-        elif reasoning_effort == "medium":
+        if reasoning_effort in get_args(REASONING_EFFORT):
             return (
-                Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
-            )
-        elif reasoning_effort == "low":
-            return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
-        elif reasoning_effort == "minimal":
-            return (
-                Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
+                Reasoning(effort=reasoning_effort, summary="detailed")
+                if auto_summary_enabled
+                else Reasoning(effort=reasoning_effort)
             )
         return None
 

litellm/constants.py

@@ -48,6 +48,9 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Data URIs exceeding this are replaced with a size placeholder.
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
+REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
+# in-memory stand-in handed to provider converters for redacted arguments; never stored
+REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
 
 MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
 
@@ -146,6 +149,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-adaptive-router-model",
     "x-litellm-applied-guardrails",
     "x-litellm-guardrail-scan-id",
+    "x-litellm-cache-key",
 ]
 
 # Gemini model-specific minimal thinking budget constants
@@ -460,6 +464,8 @@ CONNECTION_ERROR_PATTERNS: Final[list[str]] = [
 ]
 STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
+STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
+STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float(
     os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@@ -749,6 +755,7 @@ openai_compatible_endpoints: Final[list] = [
     "api.groq.com/openai/v1",
     "https://integrate.api.nvidia.com/v1",
     "api.deepseek.com/v1",
+    "api.together.ai/v1",
     "api.together.xyz/v1",
     "app.empower.dev/api/v1",
     "https://api.friendli.ai/serverless/v1",

litellm/integrations/compression_interception/handler.py

@@ -7,7 +7,7 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
 
 import time
 import uuid
-from typing import Any, Final, cast
+from typing import Any, ClassVar, Final, cast
 
 from litellm._logging import verbose_logger
 from litellm.compression import compress
@@ -72,6 +72,8 @@ class CompressionInterceptionLogger(CustomLogger):
     4. Build typed rerun plan with tool_result blocks from the compressed cache.
     """
 
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({LITELLM_CONTENT_RETRIEVE_TOOL_NAME})
+
     def __init__(
         self,
         enabled: bool = True,

litellm/integrations/custom_logger.py

@@ -3,7 +3,7 @@
 import re
 import traceback
 from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 from pydantic import BaseModel
 
@@ -60,6 +60,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
 
     enforces_request_content: bool = False
     """

litellm/integrations/langfuse/langfuse_otel.py

@@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
     LangfuseLLMObsOTELAttributes,
 )
 from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.types.integrations.langfuse_otel import (
     LangfuseSpanAttributes,
 )
@@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         )
                     elif item_type == "function_call":
                         arguments_str = getattr(item, "arguments", "{}")
-                        arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
+                        arguments_obj = (
+                            safe_json_loads(arguments_str, default={})
+                            if isinstance(arguments_str, str)
+                            else arguments_str
+                        )
                         langfuse_tool_call = {
                             "id": getattr(item, "id", ""),
                             "name": getattr(item, "name", ""),

litellm/integrations/langsmith.py

@@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
         return outputs
 
     def _ensure_required_ids(self, data: dict, run_id: str | None):
+        resolved_id: Final = run_id or str(uuid.uuid4())
         if "id" not in data or data["id"] is None:
-            run_id = str(uuid.uuid4())
-            data["id"] = run_id
+            data["id"] = resolved_id
 
-        if "trace_id" not in data or data["trace_id"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["trace_id"] = run_id
+        # LangSmith rejects the whole ingest batch unless a root run's trace_id
+        # equals the run id embedded in the first segment of dotted_order
+        posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
+            "dotted_order" not in data or data["dotted_order"] is None
+        )
+        if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
+            data["trace_id"] = resolved_id
 
         if "dotted_order" not in data or data["dotted_order"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["dotted_order"] = self.make_dot_order(run_id=run_id)
+            data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
 
     def _prepare_log_data(
         self,
@@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
             metadata = _litellm_params.get("metadata", {}) or {}
 
             fields: Final = self._extract_metadata_fields(metadata, credentials)
+            # the proxy header fan-out mirrors one value into both keys, and LangSmith
+            # rejects the whole ingest batch when run-body session_id is not an
+            # existing tracer-session uuid
+            if fields["session_id"] == fields["trace_id"]:
+                fields["session_id"] = None
             verbose_logger.debug(
                 "Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
             )

litellm/integrations/prometheus.py

@@ -96,7 +96,10 @@ class _PaginatedPrismaTable(Protocol[_TableRowT]):
 
 def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrismaTable[_TableRowT]:
     """View a repository's prisma table through the pagination surface budget metrics need."""
-    return repository.table
+    return cast(
+        _PaginatedPrismaTable[_TableRowT],
+        repository.table,  # cast-ok: prisma rows carry the budget columns the domain model declares
+    )
 
 
 class _OrgBudgetRow(Protocol):

litellm/integrations/s3_v2.py

@@ -11,6 +11,7 @@ import time
 from collections.abc import Mapping
 from datetime import datetime
 from typing import Final, cast
+from urllib.parse import quote
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
         )
 
+    def _build_object_url(self, s3_object_key: str) -> str:
+        """
+        Build the exact URL that is both signed and sent, with the key percent-encoded once.
+
+        S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
+        characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
+        the two signatures disagree (403 SignatureDoesNotMatch).
+        """
+        encoded_key: Final = quote(s3_object_key, safe="/")
+        if self.s3_endpoint_url and self.s3_bucket_name:
+            if self.s3_use_virtual_hosted_style:
+                endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
+                protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
+                return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
+            return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
+        return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
+
     def _sse_headers(self) -> Mapping[str, str]:
         candidates: Final = {
             "x-amz-server-side-encryption": self.s3_server_side_encryption,
@@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
             verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
             from botocore.credentials import Credentials
@@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 aws_region_name=self.s3_region_name,
             )
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             httpx_client: Final = _get_httpx_client(
                 params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
             )
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         try:
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
 
             verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
+            url: Final = self._build_object_url(s3_object_key)
 
             # Prepare the request for GET operation
             # For GET requests, we need x-amz-content-sha256 with hash of empty string
@@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             headers: Final = {
                 "x-amz-content-sha256": empty_string_hash,
             }
-            req: Final = requests.Request("GET", url, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
             S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            request_url: Final = prepped.url or url
-            response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
+            response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
 
             if response.status_code != 200:
                 verbose_logger.exception("S3 object not found, saw response=", response.text)

litellm/interactions/utils.py

@@ -47,6 +47,13 @@ def get_provider_interactions_api_config(
 
         return GoogleAIStudioInteractionsConfig()
 
+    if provider in (LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value):
+        from litellm.llms.vertex_ai.interactions.transformation import (
+            VertexAIInteractionsConfig,
+        )
+
+        return VertexAIInteractionsConfig()
+
     return None
 
 

litellm/litellm_core_utils/core_helpers.py

@@ -58,6 +58,26 @@ def safe_divide(
     return numerator / denominator
 
 
+def is_expected_client_error(exception: BaseException | None) -> bool:
+    """
+    True when the exception maps to an HTTP 4xx status.
+
+    ProxyException stores the status on .code (as a str), HTTPException and
+    litellm exceptions on .status_code.
+    """
+    if exception is None:
+        return False
+    code: Final[object] = getattr(exception, "code", None)
+    status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
+    if status_code is None or isinstance(status_code, bool):
+        return False
+    try:
+        status: Final = int(str(status_code))
+    except ValueError:
+        return False
+    return 400 <= status < 500
+
+
 def coerce_token_limit(value: object) -> int | None:
     """
     Coerce a max_input_tokens / max_output_tokens value to an int, treating a

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -272,6 +272,14 @@ def get_llm_provider(
                     elif endpoint == "api.deepseek.com/v1":
                         custom_llm_provider = "deepseek"
                         dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
+                    elif endpoint == "api.together.ai/v1" or endpoint == "api.together.xyz/v1":
+                        custom_llm_provider = "together_ai"
+                        dynamic_api_key = api_key or (
+                            get_secret_str("TOGETHER_API_KEY")
+                            or get_secret_str("TOGETHER_AI_API_KEY")
+                            or get_secret_str("TOGETHERAI_API_KEY")
+                            or get_secret_str("TOGETHER_AI_TOKEN")
+                        )
                     elif endpoint == "ollama.com":
                         custom_llm_provider = "ollama"
                         dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
@@ -707,7 +715,7 @@ def _get_openai_compatible_provider_info(
             dynamic_api_key,
         ) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "together_ai":
-        api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.xyz/v1"
+        api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.ai/v1"
         dynamic_api_key = api_key or (
             get_secret_str("TOGETHER_API_KEY")
             or get_secret_str("TOGETHER_AI_API_KEY")

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -172,7 +172,7 @@ def get_supported_openai_params(
         if request_type == "embeddings":
             return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "together_ai":
-        return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
+        return litellm.TogetherAIChatConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "databricks":
         if request_type == "chat_completion":
             return litellm.DatabricksConfig().get_supported_openai_params(model=model)

litellm/litellm_core_utils/json_fragment_accumulator.py

@@ -0,0 +1,97 @@
+import json
+from typing import Final, cast  # noqa: TID251  # raw_decode returns tuple[Any, int]; no cast-free unpack
+
+
+class JSONFragmentAccumulator:
+    """
+    Buffers a JSON value that arrives piecemeal over a stream (SSE data split
+    across TCP packets, one shard per network read, etc) without the O(n^2)
+    cost of repeated `buffer += fragment` string concatenation, and without
+    the O(n^2) cost of re-copying the unconsumed remainder on every peeled
+    value when one payload holds many concatenated JSON values.
+
+    Fragments are appended to a list in O(1). The buffer is only rebuilt into
+    a single string, and only decoded, when a caller asks for a value via
+    `pop_next_value`, and `could_close_json` lets callers skip that rebuild
+    entirely for fragments that plainly cannot close a JSON value yet. Once
+    rebuilt, consumed values are dropped by advancing a cursor rather than
+    slicing a new string, so draining N concatenated values already sitting
+    in the buffer costs O(n) total, not O(n^2).
+    """
+
+    def __init__(self) -> None:
+        self._chunks: list[str] = []  # mutable-ok: O(1) append; string concat would copy the buffer each time
+        self._buffer: str = (
+            ""  # mutable-ok: lazily materialized join of _chunks, rebuilt only when _chunks is non-empty
+        )
+        self._offset: int = 0  # mutable-ok: cursor past already-consumed values; avoids re-slicing on every pop
+        self._could_close: bool = False  # mutable-ok: cached heuristic; rescanning past fragments was itself O(n^2)
+
+    def __bool__(self) -> bool:
+        return bool(self._chunks) or self._offset < len(self._buffer)
+
+    def append(self, fragment: str) -> None:
+        self._chunks.append(fragment)  # mutable-ok: see __init__
+        stripped: Final = fragment.rstrip()
+        if stripped:
+            self._could_close = stripped[-1] in ("}", "]")  # mutable-ok: see __init__
+
+    def could_close_json(self) -> bool:
+        """
+        Whether the buffer's logical last non-whitespace byte is "}" or "]",
+        i.e. whether a JSON value could plausibly be complete. Tracked
+        incrementally in `append` rather than rescanned here, so a run of
+        blank keepalive fragments (e.g. from a malformed upstream stream)
+        can't make this, or the join+parse it gates, cost O(n^2).
+        """
+        return self._could_close
+
+    def _materialize(self) -> None:
+        if not self._chunks:
+            return
+        unconsumed: Final = self._buffer[self._offset :]
+        self._buffer = unconsumed + "".join(self._chunks)  # mutable-ok: merge pending fragments, once per append batch
+        self._offset = 0  # mutable-ok: see __init__
+        self._chunks = []  # mutable-ok: see __init__
+
+    def pop_next_value(self) -> tuple[bool, object]:
+        """
+        Attempt to decode one complete JSON value from the front of the
+        buffer. On success, advances a cursor past that value (keeping any
+        unconsumed tail, e.g. a second concatenated value, in place rather
+        than copying it) and returns (True, value). If the buffer is empty
+        or holds no complete value yet, it is left untouched and this
+        returns (False, None).
+        """
+        self._materialize()
+        length: Final = len(self._buffer)
+        start = self._offset
+        while start < length and self._buffer[start].isspace():
+            start += 1
+        if start >= length:
+            self._offset = start  # mutable-ok: see __init__
+            return False, None
+        decoder: Final = json.JSONDecoder()
+        try:
+            raw_value: Final = decoder.raw_decode(self._buffer, start)
+        except json.JSONDecodeError:
+            return False, None
+        decoded, end_index = cast("tuple[object, int]", raw_value)  # cast-ok: raw_decode returns tuple[Any, int]
+        self._offset = end_index  # mutable-ok: see __init__
+        if self._offset >= len(self._buffer):
+            self._buffer = ""  # mutable-ok: see __init__
+            self._offset = 0  # mutable-ok: see __init__
+            self._could_close = False  # mutable-ok: buffer is empty, nothing can close
+        return True, decoded
+
+    def snapshot(self) -> str:
+        self._materialize()
+        return self._buffer[self._offset :]
+
+    def set(self, value: str) -> None:
+        """Replace the buffer's contents with a single fragment."""
+        self._chunks = []  # mutable-ok: see __init__
+        self._buffer = value  # mutable-ok: see __init__
+        self._offset = 0  # mutable-ok: see __init__
+        stripped: Final = value.rstrip()
+        self._could_close = bool(stripped) and stripped[-1] in ("}", "]")  # mutable-ok: see __init__

litellm/litellm_core_utils/litellm_logging.py

@@ -62,7 +62,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.deepeval.deepeval import DeepEvalLogger
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
-from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
+from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     cost_breakdown_with_guardrail,
@@ -3124,6 +3124,13 @@ class Logging(LiteLLMLoggingBaseClass):
         if not hasattr(self, "model_call_details"):
             self.model_call_details = {}
 
+        if (
+            self.model_call_details.get("log_event_type") == "failed_api_call"
+            and self.model_call_details.get("exception") is exception
+            and self.model_call_details.get("standard_logging_object") is not None
+        ):
+            return start_time, self.model_call_details["end_time"]
+
         self.model_call_details["log_event_type"] = "failed_api_call"
         self.model_call_details["exception"] = exception
         self.model_call_details["traceback_exception"] = (
@@ -5455,9 +5462,10 @@ class StandardLoggingPayloadSetup:
         error_class: Final[str] = str(original_exception.__class__.__name__) if original_exception else ""
         _llm_provider_in_exception: Final = getattr(original_exception, "llm_provider", "")
 
-        # Get traceback information (first 100 lines)
         traceback_info = traceback_str or ""
-        if original_exception:
+        if original_exception and (
+            litellm.log_client_error_tracebacks or not is_expected_client_error(original_exception)
+        ):
             tb: Final[TracebackType | None] = getattr(original_exception, "__traceback__", None)
             if tb:
                 tb_lines: Final = traceback.format_tb(tb)
@@ -5930,11 +5938,15 @@ def get_standard_logging_object_payload(
             response_model_name = final_response_obj.get("model")
 
         # For Azure Model Router, preserve the actual model in the top-level standard
-        # logging payload only when the user has opted in.
+        # logging payload.
+        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
         requested_model: Final = kwargs.get("model")
-        if (
-            isinstance(requested_model, str)
-            and ("model_router" in requested_model.lower() or "model-router" in requested_model.lower())
+        stamped_selected_model: Final = AzureFoundryModelInfo.get_model_router_selected_model(hidden_params)
+        if stamped_selected_model is not None:
+            model_name = stamped_selected_model
+        elif (
+            AzureFoundryModelInfo.is_model_router_call(model=requested_model, hidden_params=hidden_params)
             and isinstance(response_model_name, str)
             and response_model_name
         ):

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -470,6 +470,8 @@ def update_messages_with_model_file_ids(
     from litellm.proxy.openai_files_endpoints.common_utils import (
         _is_base64_encoded_unified_file_id,
         convert_b64_uid_to_unified_uid,
+        get_original_file_id,
+        is_model_embedded_id,
     )
 
     for message in messages:
@@ -508,6 +510,11 @@ def update_messages_with_model_file_ids(
                                 unified_file_id = convert_b64_uid_to_unified_uid(file_id)
                                 if "llm_output_file_id," in unified_file_id:
                                     provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
+                            if not provider_file_id and is_model_embedded_id(file_id):
+                                # `litellm:<raw_id>;model,<m>` encoding from the
+                                # x-litellm-model upload path. Strip the wrapper
+                                # so the provider sees its own ID.
+                                provider_file_id = get_original_file_id(file_id)
                             file_object_file_field["file_id"] = provider_file_id or file_id
                         if format:
                             file_object_file_field["format"] = format
@@ -535,6 +542,8 @@ def update_responses_input_with_model_file_ids(
     from litellm.proxy.openai_files_endpoints.common_utils import (
         _is_base64_encoded_unified_file_id,
         convert_b64_uid_to_unified_uid,
+        get_original_file_id,
+        is_model_embedded_id,
     )
 
     if isinstance(input, str):
@@ -578,6 +587,13 @@ def update_responses_input_with_model_file_ids(
                                 updated_content_item = content_item.copy()
                                 updated_content_item["file_id"] = provider_file_id
                                 updated_content.append(updated_content_item)
+                            elif is_model_embedded_id(file_id):
+                                # `litellm:<raw_id>;model,<m>` encoding from the
+                                # x-litellm-model upload path. Strip the wrapper
+                                # so the provider sees its own ID.
+                                updated_content_item = content_item.copy()
+                                updated_content_item["file_id"] = get_original_file_id(file_id)
+                                updated_content.append(updated_content_item)
                             else:
                                 # Not a managed file, keep as-is
                                 updated_content.append(content_item)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -16,6 +16,7 @@ import litellm.types
 import litellm.types.llms
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.constants import REDACTED_BY_LITELLM
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
 from litellm.types.files import get_file_extension_from_mime_type
@@ -642,49 +643,6 @@ def claude_2_1_pt(
     return prompt
 
 
-### TOGETHER AI
-
-
-def get_model_info(token, model):
-    try:
-        headers: Final = {"Authorization": f"Bearer {token}"}
-        client: Final = HTTPHandler(concurrent_limit=1)
-        response: Final = client.get("https://api.together.xyz/models/info", headers=headers)
-        if response.status_code == 200:
-            model_info: Final = response.json()
-            for m in model_info:
-                if m["name"].lower().strip() == model.strip():
-                    return m["config"].get("prompt_format", None), m["config"].get("chat_template", None)
-            return None, None
-        else:
-            return None, None
-    except Exception:  # safely fail a prompt template request
-        return None, None
-
-
-## OLD TOGETHER AI FLOW
-# def format_prompt_togetherai(messages, prompt_format, chat_template):
-#     if prompt_format is None:
-#         return default_pt(messages)
-
-#     human_prompt, assistant_prompt = prompt_format.split("{prompt}")
-
-#     if chat_template is not None:
-#         prompt = hf_chat_template(
-#             model=None, messages=messages, chat_template=chat_template
-#         )
-#     elif prompt_format is not None:
-#         prompt = custom_prompt(
-#             role_dict={},
-#             messages=messages,
-#             initial_prompt_value=human_prompt,
-#             final_prompt_value=assistant_prompt,
-#         )
-#     else:
-#         prompt = default_pt(messages)
-#     return prompt
-
-
 ### IBM Granite
 
 
@@ -5383,12 +5341,13 @@ def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) ->
         return raw
     if not isinstance(raw, str):
         return {}
+    normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
     from litellm.litellm_core_utils.prompt_templates.common_utils import (
         parse_tool_call_arguments,
     )
 
     try:
-        parsed: Final = parse_tool_call_arguments(raw, tool_name=tool_name, context=context)
+        parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
     except ValueError as e:
         verbose_logger.warning("Failed to parse tool call arguments: %s", e)
         return {}

litellm/litellm_core_utils/redact_messages.py

@@ -13,6 +13,7 @@ import inspect
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
+from litellm.constants import REDACTED_BY_LITELLM
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
@@ -84,29 +85,31 @@ def _redact_tool_calls(tool_calls) -> None:
     for tool_call in tool_calls:
         function = getattr(tool_call, "function", None)
         if function is not None and hasattr(function, "arguments"):
-            function.arguments = "redacted-by-litellm"
+            function.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_function_call(function_call) -> None:
     """Redact legacy assistant function_call arguments."""
     if function_call is not None and hasattr(function_call, "arguments"):
-        function_call.arguments = "redacted-by-litellm"
+        function_call.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
     if isinstance(choice, litellm.Choices):
-        choice.message.content = "redacted-by-litellm"
-        if hasattr(choice.message, "reasoning_content"):
-            choice.message.reasoning_content = "redacted-by-litellm"
+        if choice.message.content is not None:
+            choice.message.content = REDACTED_BY_LITELLM
+        if getattr(choice.message, "reasoning_content", None) is not None:
+            choice.message.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
         _redact_tool_calls(getattr(choice.message, "tool_calls", None))
         _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
-        choice.delta.content = "redacted-by-litellm"
-        if hasattr(choice.delta, "reasoning_content"):
-            choice.delta.reasoning_content = "redacted-by-litellm"
+        if choice.delta.content is not None:
+            choice.delta.content = REDACTED_BY_LITELLM
+        if getattr(choice.delta, "reasoning_content", None) is not None:
+            choice.delta.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
         _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
@@ -116,23 +119,23 @@ def _redact_choice_content(choice):
 def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
-        if hasattr(output_item, "text"):
-            output_item.text = "redacted-by-litellm"
+        if getattr(output_item, "text", None) is not None:
+            output_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "content") and isinstance(output_item.content, list):
             for content_part in output_item.content:
-                if hasattr(content_part, "text"):
-                    content_part.text = "redacted-by-litellm"
+                if getattr(content_part, "text", None) is not None:
+                    content_part.text = REDACTED_BY_LITELLM
 
         # Redact reasoning items in output array
         if hasattr(output_item, "type") and output_item.type == "reasoning":
             if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
                 for summary_item in output_item.summary:
-                    if hasattr(summary_item, "text"):
-                        summary_item.text = "redacted-by-litellm"
+                    if getattr(summary_item, "text", None) is not None:
+                        summary_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
-            output_item.arguments = "redacted-by-litellm"
+            output_item.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_responses_api_output_dict(output_items, redacted_str: str):
@@ -141,17 +144,17 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
         if not isinstance(output_item, dict):
             continue
 
-        if "text" in output_item:
+        if output_item.get("text") is not None:
             output_item["text"] = redacted_str
 
         if isinstance(output_item.get("content"), list):
             for content_item in output_item["content"]:
-                if isinstance(content_item, dict) and "text" in content_item:
+                if isinstance(content_item, dict) and content_item.get("text") is not None:
                     content_item["text"] = redacted_str
 
         if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
             for summary_item in output_item["summary"]:
-                if isinstance(summary_item, dict) and "text" in summary_item:
+                if isinstance(summary_item, dict) and summary_item.get("text") is not None:
                     summary_item["text"] = redacted_str
 
         if output_item.get("type") == "function_call" and "arguments" in output_item:
@@ -164,7 +167,7 @@ def _redact_standard_logging_object(model_call_details: dict):
     if standard_logging_object is None:
         return
 
-    redacted_str: Final = "redacted-by-litellm"
+    redacted_str: Final = REDACTED_BY_LITELLM
 
     if standard_logging_object.get("messages") is not None:
         standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
@@ -188,40 +191,42 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
-def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
+def _redact_tool_calls_dict(message: dict) -> None:
     """Redact tool call / function_call arguments in a dict-form message or delta."""
     tool_calls: Final = message.get("tool_calls")
     if isinstance(tool_calls, list):
         for tool_call in tool_calls:
             if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
-                tool_call["function"]["arguments"] = redacted_str
+                tool_call["function"]["arguments"] = REDACTED_BY_LITELLM
 
     function_call: Final = message.get("function_call")
     if isinstance(function_call, dict) and "arguments" in function_call:
-        function_call["arguments"] = redacted_str
+        function_call["arguments"] = REDACTED_BY_LITELLM
 
 
 def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
             if "message" in choice and isinstance(choice["message"], dict):
-                choice["message"]["content"] = redacted_str
-                if "reasoning_content" in choice["message"]:
+                if choice["message"].get("content") is not None:
+                    choice["message"]["content"] = redacted_str
+                if choice["message"].get("reasoning_content") is not None:
                     choice["message"]["reasoning_content"] = redacted_str
                 if "thinking_blocks" in choice["message"]:
                     choice["message"]["thinking_blocks"] = None
                 if "audio" in choice["message"]:
                     choice["message"]["audio"] = None
-                _redact_tool_calls_dict(choice["message"], redacted_str)
+                _redact_tool_calls_dict(choice["message"])
             elif "delta" in choice and isinstance(choice["delta"], dict):
-                choice["delta"]["content"] = redacted_str
-                if "reasoning_content" in choice["delta"]:
+                if choice["delta"].get("content") is not None:
+                    choice["delta"]["content"] = redacted_str
+                if choice["delta"].get("reasoning_content") is not None:
                     choice["delta"]["reasoning_content"] = redacted_str
                 if "thinking_blocks" in choice["delta"]:
                     choice["delta"]["thinking_blocks"] = None
                 if "audio" in choice["delta"]:
                     choice["delta"]["audio"] = None
-                _redact_tool_calls_dict(choice["delta"], redacted_str)
+                _redact_tool_calls_dict(choice["delta"])
         else:
             _redact_choice_content(choice)
 
@@ -235,7 +240,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
     copy via redact_streaming_responses_for_custom_logger instead.
     """
     # Redact model_call_details
-    model_call_details["messages"] = [{"role": "user", "content": "redacted-by-litellm"}]
+    model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""
     _redact_standard_logging_object(model_call_details)
@@ -256,13 +261,13 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             or hasattr(result, "__anext__")  # async generator
         ):  # async iterator
             # For async objects, return a simple redacted response without deepcopy
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
 
         if not (
             isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
             or (isinstance(result, dict) and ("choices" in result or "output" in result))
         ):
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
 
         _result: Final = copy.deepcopy(result)
         if isinstance(_result, litellm.ModelResponse):
@@ -273,11 +278,11 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
         elif isinstance(_result, dict) and "choices" in _result:
             # Handle dict representation of ModelResponse (e.g., from model_dump())
             if _result.get("choices") is not None:
-                _redact_model_response_dict_choices(_result["choices"], "redacted-by-litellm")
+                _redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
             redact_vertex_ai_metadata_from_logged_object(_result)
         elif isinstance(_result, dict) and "output" in _result:
             if isinstance(_result.get("output"), list):
-                _redact_responses_api_output_dict(_result["output"], "redacted-by-litellm")
+                _redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
         elif isinstance(_result, litellm.ResponsesAPIResponse):
             if hasattr(_result, "output"):
                 _redact_responses_api_output(_result.output)
@@ -288,7 +293,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             if hasattr(_result, "data") and _result.data is not None:
                 _result.data = []
         else:
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
         return _result
 
 

litellm/llms/anthropic/chat/handler.py

@@ -18,6 +18,7 @@ from litellm.anthropic_beta_headers_manager import (
 )
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -654,7 +655,7 @@ class ModelResponseIterator:
 
         # For handling partial JSON chunks from fragmentation
         # See: https://github.com/BerriAI/litellm/issues/17473
-        self.accumulated_json: str = ""
+        self._json_buffer = JSONFragmentAccumulator()
         self.chunk_type: Literal["valid_json", "accumulated_json"] = "valid_json"
 
         # Track current content block type to avoid emitting tool calls for non-tool blocks
@@ -678,6 +679,14 @@ class ModelResponseIterator:
         self._current_server_tool_id: str | None = None
         self._container_id: str | None = None
 
+    @property
+    def accumulated_json(self) -> str:
+        return self._json_buffer.snapshot()
+
+    @accumulated_json.setter
+    def accumulated_json(self, value: str) -> None:
+        self._json_buffer.set(value)
+
     def check_empty_tool_call_args(self) -> bool:
         """
         Check if the tool call block so far has been an empty string
@@ -1149,31 +1158,39 @@ class ModelResponseIterator:
         container: Final = message_delta["delta"].get("container")
         return finish_reason, usage, container
 
-    def _handle_accumulated_json_chunk(self, data_str: str) -> ModelResponseStream | None:
+    def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
         """
         Handle partial JSON chunks by accumulating them until valid JSON is received.
 
         This fixes network fragmentation issues where SSE data chunks may be split
         across TCP packets. See: https://github.com/BerriAI/litellm/issues/17473
 
+        Mid-stream, defer parsing until the buffer's last byte can close a value:
+        attempting a parse after every fragment of one large object is O(n^2) and
+        holds the GIL, freezing the event loop. At end of stream (is_final) no more
+        data is coming, so drain whatever complete values remain regardless of the
+        trailing byte.
+
         Args:
             data_str: The JSON string to parse (without "data:" prefix)
+            is_final: True when called from the end-of-stream drain, where the
+                trailing-byte heuristic no longer applies
 
         Returns:
             ModelResponseStream if JSON is complete, None if still accumulating
         """
-        # Accumulate JSON data
-        self.accumulated_json += data_str
+        self._json_buffer.append(data_str)
 
-        # Try to parse the accumulated JSON
-        try:
-            data_json: Final = json.loads(self.accumulated_json)
-            self.accumulated_json = ""  # Reset after successful parsing
-            return self.chunk_parser(chunk=data_json)
-        except json.JSONDecodeError:
-            # If it's not valid JSON yet, continue to the next chunk
+        if not is_final and not self._json_buffer.could_close_json():
             return None
 
+        while True:
+            found, decoded = self._json_buffer.pop_next_value()
+            if not found:
+                return None
+            if isinstance(decoded, dict):
+                return self.chunk_parser(chunk=decoded)
+
     def _parse_sse_data(self, str_line: str) -> ModelResponseStream | None:
         """
         Parse SSE data line, handling both complete and partial JSON chunks.
@@ -1209,13 +1226,10 @@ class ModelResponseIterator:
                 chunk = self.response_iterator.__next__()
             except StopIteration:
                 # If we have accumulated JSON when stream ends, try to parse it
-                if self.accumulated_json:
-                    try:
-                        data_json = json.loads(self.accumulated_json)
-                        self.accumulated_json = ""
-                        return self.chunk_parser(chunk=data_json)
-                    except json.JSONDecodeError:
-                        pass
+                if self._json_buffer:
+                    result = self._handle_accumulated_json_chunk(data_str="", is_final=True)
+                    if result is not None:
+                        return result
                 raise StopIteration
             except ValueError as e:
                 raise RuntimeError(f"Error receiving chunk from stream: {e}")
@@ -1258,13 +1272,10 @@ class ModelResponseIterator:
                 chunk = await self.async_response_iterator.__anext__()
             except StopAsyncIteration:
                 # If we have accumulated JSON when stream ends, try to parse it
-                if self.accumulated_json:
-                    try:
-                        data_json = json.loads(self.accumulated_json)
-                        self.accumulated_json = ""
-                        return self.chunk_parser(chunk=data_json)
-                    except json.JSONDecodeError:
-                        pass
+                if self._json_buffer:
+                    result = self._handle_accumulated_json_chunk(data_str="", is_final=True)
+                    if result is not None:
+                        return result
                 raise StopAsyncIteration
             except ValueError as e:
                 raise RuntimeError(f"Error receiving chunk from stream: {e}")

litellm/llms/anthropic/chat/transformation.py

@@ -1217,8 +1217,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if reasoning_effort is None or reasoning_effort == "none":
             return None
         if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
+            # without display, Anthropic defaults adaptive thinking to
+            # display="omitted" and returns a blank thinking block
             return AnthropicThinkingParam(
                 type="adaptive",
+                display="summarized",
             )
         elif reasoning_effort == "low":
             return AnthropicThinkingParam(
@@ -2146,7 +2149,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         )
 
     @staticmethod
-    def _thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None:
+    def thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None:
         details: Final = usage_object.get("output_tokens_details")
         if not isinstance(details, Mapping):
             return None
@@ -2178,7 +2181,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         reported_thinking_tokens: Final = (
             iteration_thinking_tokens
             if iteration_thinking_tokens is not None
-            else self._thinking_tokens_from_usage(usage_object)
+            else self.thinking_tokens_from_usage(usage_object)
         )
         if reported_thinking_tokens is not None:
             capped_reported: Final = min(max(0, reported_thinking_tokens), completion_tokens)
@@ -2201,7 +2204,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def _sum_iteration_thinking_tokens(self, iterations: Sequence[object]) -> int | None:
         per_iteration: Final = tuple(
-            self._thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None
+            self.thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None
             for iteration in iterations
         )
         reported: Final = tuple(tokens for tokens in per_iteration if tokens is not None)

litellm/llms/anthropic/common_utils.py

@@ -44,6 +44,21 @@ DROP_DISABLED_THINKING_WARNING: Final = (
     "thinking blocks, and those thinking tokens are billed as output tokens."
 )
 
+# Anthropic error `type` (both the JSON error body and SSE `event: error`
+# payloads use this field) mapped to the HTTP status code it corresponds to.
+ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = MappingProxyType(
+    {
+        "invalid_request_error": 400,
+        "authentication_error": 401,
+        "permission_error": 403,
+        "not_found_error": 404,
+        "rate_limit_error": 429,
+        "api_error": 500,
+        "overloaded_error": 503,
+        "timeout_error": 504,
+    }
+)
+
 _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -434,7 +434,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                 content_items = list(content.get("content", []))
 
                                 # Single-item text keeps the backward-compatible string format; a single
-                                # image becomes a structured image_url part
+                                # image or document becomes a structured image_url part
                                 if len(content_items) == 1:
                                     c = content_items[0]
                                     if isinstance(c, str):
@@ -454,7 +454,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                             )
                                             self._add_cache_control_if_applicable(content, tool_result, model)
                                             tool_message_list.append(tool_result)
-                                        elif c.get("type") == "image":
+                                        elif c.get("type") in ("image", "document"):
                                             image_part = self._tool_result_image_part(c.get("source"))
                                             tool_result = ChatCompletionToolMessage(
                                                 role="tool",
@@ -482,7 +482,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                                         text=c.get("text", ""),
                                                     )
                                                 )
-                                            elif c.get("type") == "image":
+                                            elif c.get("type") in ("image", "document"):
                                                 image_part = self._tool_result_image_part(c.get("source"))
                                                 if image_part:
                                                     combined_content_parts.append(image_part)

litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py

@@ -6,13 +6,41 @@ yields every chunk to the caller (preserving real streaming), collects
 all bytes, and on stream exhaustion rebuilds the full Anthropic response
 to run through agentic completion hooks. If an agentic hook fires, the
 follow-up response is chained as Phase 2 of the same iterator.
+
+In hold-back mode (``hold_back=True``) chunks are buffered instead of yielded
+live, keepalive pings run whenever no other byte is ready, and then either the
+follow-up replaces the message or the buffer replays, except that a tool_use for
+a server-fulfilled tool fails the turn rather than reaching a client that cannot
+execute it.
 """
 
+import asyncio
+import contextlib
 import json
 from collections.abc import AsyncIterator
 from typing import Any, Final, cast
 
 from litellm._logging import verbose_logger
+from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
+
+HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0
+SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
+    b"event: error\n"
+    b'data: {"type": "error", "error": {"type": "api_error", "message": '
+    b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
+)
+
+
+def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
+    return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
+
+
+async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
+    try:
+        return await iterator.__anext__()
+    except StopAsyncIteration:
+        return None
+
 
 # ---------------------------------------------------------------------------
 # SSE parsing helpers (module-level to keep the class lean)
@@ -156,6 +184,9 @@ class AgenticAnthropicStreamingIterator:
         logging_obj: Any,
         custom_llm_provider: str,
         kwargs: dict,
+        hold_back: bool = False,
+        server_fulfilled_tool_names: frozenset[str] = frozenset(),
+        ping_interval_seconds: float = HOLD_BACK_PING_INTERVAL_SECONDS,
     ):
         self._inner = completion_stream.__aiter__()
         self._http_handler = http_handler
@@ -166,16 +197,32 @@ class AgenticAnthropicStreamingIterator:
         self._logging_obj = logging_obj
         self._custom_llm_provider = custom_llm_provider
         self._kwargs = kwargs
+        self._hold_back = hold_back
+        self._server_fulfilled_tool_names = server_fulfilled_tool_names
+        self._ping_interval_seconds = ping_interval_seconds
 
         self._collected_bytes: list[bytes] = []
         self._stream_exhausted = False
         self._hook_processing_done = False
         self._follow_up_iterator: AsyncIterator | None = None
+        self._drain_task: asyncio.Task | None = None
+        self._hook_task: asyncio.Task | None = None
+        self._follow_up_chunk_task: asyncio.Task | None = None
+        self._replay_index = 0
+        self._error_emitted = False
+
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        """Whether provider output was received but withheld from the client behind keepalive pings."""
+        return self._hold_back and bool(self._collected_bytes)
 
     def __aiter__(self):
         return self
 
     async def __anext__(self) -> bytes:
+        if self._hold_back:
+            return await self._anext_held_back()
+
         # Phase 1: yield from upstream, collect bytes
         if not self._stream_exhausted:
             try:
@@ -194,11 +241,102 @@ class AgenticAnthropicStreamingIterator:
 
         raise StopAsyncIteration
 
+    async def _drain_upstream(self) -> None:
+        try:
+            while True:
+                self._collected_bytes.append(await self._inner.__anext__())
+        except StopAsyncIteration:
+            return
+
+    async def _completed_within_ping_interval(self, task: asyncio.Task) -> bool:
+        try:
+            await asyncio.wait_for(asyncio.shield(task), timeout=self._ping_interval_seconds)
+        except asyncio.TimeoutError:
+            return False
+        return True
+
+    async def _anext_held_back(self) -> bytes:
+        if self._drain_task is None:
+            self._drain_task = asyncio.create_task(self._drain_upstream())
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
+
+        if not self._stream_exhausted:
+            if not await self._completed_within_ping_interval(self._drain_task):
+                return STREAM_SSE_KEEPALIVE_PING_BYTES
+            self._stream_exhausted = True
+
+        if self._hook_task is None:
+            self._hook_task = asyncio.create_task(self._process_agentic_hooks())
+        if not await self._completed_within_ping_interval(self._hook_task):
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
+
+        if self._follow_up_iterator is not None:
+            return await self._next_follow_up_chunk(self._follow_up_iterator)
+
+        if self._buffer_holds_server_fulfilled_tool_use():
+            if self._error_emitted:
+                raise StopAsyncIteration
+            self._error_emitted = True
+            verbose_logger.error(
+                "AgenticStreamingIterator: hooks did not replace a message containing a server-fulfilled "
+                "tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client",
+                self._model,
+            )
+            return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
+
+        if self._replay_index < len(self._collected_bytes):
+            chunk: Final = self._collected_bytes[self._replay_index]
+            self._replay_index += 1
+            return chunk
+
+        raise StopAsyncIteration
+
+    async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes:
+        if self._follow_up_chunk_task is None:
+            self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
+        if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
+        chunk: Final = self._follow_up_chunk_task.result()
+        self._follow_up_chunk_task = None
+        if chunk is None:
+            raise StopAsyncIteration
+        return chunk
+
+    def _buffer_holds_server_fulfilled_tool_use(self) -> bool:
+        if not self._server_fulfilled_tool_names:
+            return False
+        started_blocks: Final = (
+            data.get("content_block")
+            for event_type, data in _parse_sse_events(b"".join(self._collected_bytes))
+            if event_type == "content_block_start"
+        )
+        return any(
+            isinstance(block, dict)
+            and block.get("type") == "tool_use"
+            and block.get("name") in self._server_fulfilled_tool_names
+            for block in started_blocks
+        )
+
+    @staticmethod
+    async def _settle_task(task: asyncio.Task | None) -> None:
+        if task is None:
+            return
+        if task.done():
+            if not task.cancelled():
+                task.exception()
+            return
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+
     async def aclose(self) -> None:
         from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
             aclose_if_supported,
         )
 
+        await self._settle_task(self._drain_task)
+        await self._settle_task(self._hook_task)
+        await self._settle_task(self._follow_up_chunk_task)
         await aclose_if_supported(self._inner)
         await aclose_if_supported(self._follow_up_iterator)
 
@@ -217,11 +355,6 @@ class AgenticAnthropicStreamingIterator:
                 verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
                 return
 
-            [
-                (f"{b.get('type')}({b.get('name', '')})" if b.get("type") == "tool_use" else b.get("type"))
-                for b in rebuilt.get("content", [])
-            ]
-
             result: Final = await self._http_handler._call_agentic_completion_hooks(
                 response=rebuilt,
                 model=self._model,

litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py

@@ -46,6 +46,10 @@ class AnthropicMessagesStreamCacheWriter:
             stream._hidden_params if isinstance(stream, AnthropicMessagesStreamingResponse) else _EMPTY_MAPPING
         )
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self.stream, "has_buffered_provider_output", False) is True
+
     def __aiter__(self) -> "AnthropicMessagesStreamCacheWriter":
         return self
 

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -1,6 +1,6 @@
 import asyncio
 import json
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from datetime import datetime
 from typing import Any, Final, Protocol, runtime_checkable
 
@@ -11,9 +11,11 @@ from typing_extensions import TypedDict
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
 from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
 from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
@@ -33,26 +35,239 @@ def _is_message_stop_chunk(chunk: object) -> bool:
     return False
 
 
-def _is_provider_error_chunk(chunk: object) -> bool:
+def is_anthropic_ping_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk is a pure ``ping`` keepalive frame. It carries no content
+    and can recur indefinitely on a slow-starting or idle connection, so a
+    mid-stream fallback wrapper drops it outright while still deciding
+    whether to commit to the primary stream, rather than buffering it.
+
+    A physical transport chunk that coalesces a ping with any other SSE
+    event (``message_start``, ``content_block_delta``, ``event: error``, ...)
+    is NOT a pure ping - dropping it whole would discard those events - so
+    only a chunk whose every ``event:`` line is ``event: ping`` qualifies.
+    """
     if isinstance(chunk, dict):
-        return chunk.get("type") == "error"
+        return chunk.get("type") == "ping"
     if isinstance(chunk, (bytes, bytearray)):
-        return any(line == b"event: error" for line in chunk.splitlines())
+        event_lines: Final = tuple(line for line in chunk.splitlines() if line.startswith(b"event:"))
+        return bool(event_lines) and all(line == b"event: ping" for line in event_lines)
     return False
 
 
+def is_anthropic_content_delta_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk carries actual assistant-generated output (a
+    ``content_block_delta`` frame), as opposed to a lifecycle/bookkeeping
+    frame (``message_start``, ``content_block_start``/``stop``,
+    ``message_delta``, ``message_stop``, ``ping``) that carries nothing
+    worth preserving before an invisible mid-stream fallback retry.
+    """
+    if isinstance(chunk, dict):
+        return chunk.get("type") == "content_block_delta"
+    if isinstance(chunk, (bytes, bytearray)):
+        return any(line == b"event: content_block_delta" for line in chunk.splitlines())
+    return False
+
+
+def _decoded_sse_data_line(line: bytes) -> object | None:
+    if not line.startswith(b"data:"):
+        return None
+    try:
+        return json.loads(line[len(b"data:") :].strip())
+    except (ValueError, TypeError):
+        return None
+
+
+def _anthropic_error_event_payload(chunk: object) -> Mapping[str, object] | None:
+    if isinstance(chunk, dict):
+        return chunk if chunk.get("type") == "error" else None
+    if isinstance(chunk, (bytes, bytearray)):
+        decoded_lines: Final = (_decoded_sse_data_line(line) for line in chunk.splitlines())
+        return next(
+            (
+                candidate
+                for candidate in decoded_lines
+                if isinstance(candidate, dict) and candidate.get("type") == "error"
+            ),
+            None,
+        )
+    return None
+
+
+def _anthropic_error_body(chunk: object) -> Mapping[str, object] | None:
+    """Return the ``error`` object of an Anthropic SSE ``event: error`` chunk, or None."""
+    payload: Final = _anthropic_error_event_payload(chunk)
+    error_body: Final = payload.get("error") if payload is not None else None
+    return error_body if isinstance(error_body, dict) else None
+
+
+def _is_provider_error_chunk(chunk: object) -> bool:
+    return _anthropic_error_body(chunk) is not None
+
+
+def parse_anthropic_error_event(chunk: object) -> tuple[str, str, int] | None:
+    """
+    Extract ``(error_type, message, http_status_code)`` from an Anthropic SSE
+    ``event: error`` chunk (raw bytes or an already-decoded dict), or None if
+    ``chunk`` is not an error event.
+
+    The status code is looked up via ANTHROPIC_ERROR_STATUS_CODE_MAP,
+    defaulting to 500 for an error ``type`` Anthropic hasn't documented yet.
+    """
+    error_body: Final = _anthropic_error_body(chunk)
+    if error_body is None:
+        return None
+    error_type: Final = error_body.get("type")
+    if not isinstance(error_type, str):
+        return None
+    message: Final = error_body.get("message")
+    return (
+        error_type,
+        message if isinstance(message, str) else error_type,
+        ANTHROPIC_ERROR_STATUS_CODE_MAP.get(error_type, 500),
+    )
+
+
 def _is_terminal_stream_chunk(chunk: object) -> bool:
     return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
 
 
+def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
+    return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
 def _incomplete_stream_error_sse_event() -> bytes:
-    payload: Final = json.dumps(
-        {
-            "type": "error",
-            "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
-        }
+    return _sse_event(  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "error",
+        {"type": "error", "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE}},
+    )
+
+
+def _anthropic_content_block_start_and_deltas(
+    block: Mapping[str, object],
+) -> tuple[Mapping[str, object], tuple[Mapping[str, object], ...]]:
+    """
+    ``(content_block_start.content_block, content_block_delta.delta events)``
+    for one Anthropic response content block. A thinking block emits both a
+    thinking_delta and a trailing signature_delta - a real Anthropic stream
+    does the same, and dropping the signature makes any replay of that
+    assistant message (a follow-up turn, a tool-use continuation) fail
+    Anthropic's thinking-signature verification. redacted_thinking has no
+    delta at all - it is sent complete in content_block_start.
+    """
+    match block.get("type"):
+        case "tool_use":
+            return (
+                {  # mutable-ok: one-shot payload
+                    "id": block.get("id"),
+                    "name": block.get("name"),
+                    "input": {},  # mutable-ok: one-shot payload
+                    "type": "tool_use",
+                },
+                (
+                    {  # mutable-ok: one-shot payload
+                        "partial_json": json.dumps(block.get("input") or {}),  # mutable-ok: one-shot payload
+                        "type": "input_json_delta",
+                    },
+                ),
+            )
+        case "thinking":
+            signature: Final = block.get("signature")
+            signature_deltas: Final = (
+                ({"signature": signature, "type": "signature_delta"},)  # mutable-ok: one-shot payload
+                if isinstance(signature, str) and signature
+                else ()
+            )
+            return (
+                {"thinking": "", "signature": "", "type": "thinking"},  # mutable-ok: one-shot payload
+                (
+                    {"thinking": block.get("thinking") or "", "type": "thinking_delta"},  # mutable-ok: one-shot payload
+                    *signature_deltas,
+                ),
+            )
+        case "redacted_thinking":
+            return ({"type": "redacted_thinking", "data": block.get("data")}, ())  # mutable-ok: one-shot JSON payload
+        case _:
+            return (
+                {"type": "text", "text": ""},  # mutable-ok: one-shot JSON payload
+                ({"type": "text_delta", "text": block.get("text") or ""},),  # mutable-ok: one-shot JSON payload
+            )
+
+
+def anthropic_messages_response_as_sse_events(response: AnthropicMessagesResponse) -> tuple[bytes, ...]:
+    """
+    Render a complete (non-streaming) AnthropicMessagesResponse as the SSE
+    event sequence a real streaming request would have produced.
+
+    A mid-stream fallback can resolve to a non-streaming response even
+    though the client asked to stream (e.g. an agentic tool-use loop that
+    intercepts and returns a complete message) - yielding that dict directly
+    into a `/v1/messages` SSE byte stream would produce a malformed
+    response, so it's synthesized into the message_start/content_block_*/
+    message_delta/message_stop lifecycle a real stream would have sent.
+    """
+    content_blocks: Final = response.get("content") or ()
+    content_events: Final = (
+        event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
+    )
+    # A real message_start always carries a null stop_reason/stop_sequence and
+    # a zero output_tokens - those are only known once generation finishes, so
+    # copying the completed response's final values here would let a client
+    # treat the message as already finished, or double-count output tokens.
+    message_start_usage: Final = {  # mutable-ok: one-shot JSON payload
+        **(response.get("usage") or {}),
+        "output_tokens": 0,
+    }
+    message_start_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_start",
+        "message": {  # mutable-ok: one-shot JSON payload
+            **response,
+            "content": [],  # mutable-ok: one-shot JSON payload
+            "stop_reason": None,
+            "stop_sequence": None,
+            "usage": message_start_usage,
+        },
+    }
+    message_delta_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_delta",
+        "delta": {  # mutable-ok: one-shot JSON payload
+            "stop_reason": response.get("stop_reason"),
+            "stop_sequence": response.get("stop_sequence"),
+        },
+        "usage": response.get("usage") or {},  # mutable-ok: one-shot JSON payload
+    }
+    return (
+        _sse_event("message_start", message_start_payload),
+        *content_events,
+        _sse_event("message_delta", message_delta_payload),
+        _sse_event("message_stop", {"type": "message_stop"}),  # mutable-ok: one-shot JSON payload
+    )
+
+
+def _anthropic_content_block_events(index: int, block: Mapping[str, object]) -> tuple[bytes, ...]:
+    start_block, deltas = _anthropic_content_block_start_and_deltas(block)
+    start_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_start",
+        "index": index,
+        "content_block": start_block,
+    }
+    stop_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_stop",
+        "index": index,
+    }
+    delta_events: Final = tuple(
+        _sse_event(
+            "content_block_delta",
+            {"type": "content_block_delta", "index": index, "delta": delta},  # mutable-ok: one-shot payload
+        )
+        for delta in deltas
+    )
+    return (
+        _sse_event("content_block_start", start_payload),
+        *delta_events,
+        _sse_event("content_block_stop", stop_payload),
     )
-    return f"event: error\ndata: {payload}\n\n".encode()
 
 
 class AnthropicMessagesStreamHiddenParams(TypedDict):
@@ -97,6 +312,10 @@ class AnthropicMessagesStreamingResponse:
         self.completion_stream = completion_stream
         self._hidden_params = hidden_params
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self.completion_stream, "has_buffered_provider_output", False) is True
+
     def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
         return self
 

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -87,6 +87,51 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             return source.get("url")
         return None
 
+    @staticmethod
+    def _translate_anthropic_document_block_to_file_part(
+        block: Mapping[str, object],
+    ) -> dict[str, str] | None:  # mutable-ok: API message payload
+        """Convert an Anthropic document block to a Responses input_file part."""
+        raw_source: Final = block.get("source")
+        if not isinstance(raw_source, Mapping):
+            return None
+        source: Final = cast(Mapping[str, object], raw_source)  # cast-ok: untrusted client payload
+        source_type: Final = source.get("type")
+        if source_type == "base64":
+            data: Final = source.get("data")
+            if not isinstance(data, str) or not data:
+                return None
+            raw_media_type: Final = source.get("media_type")
+            media_type: Final = (
+                raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf"
+            )
+            raw_title: Final = block.get("title")
+            filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf"
+            return {  # mutable-ok: API message payload
+                "type": "input_file",
+                "filename": filename,
+                "file_data": f"data:{media_type};base64,{data}",
+            }
+        if source_type == "url":
+            url: Final = source.get("url")
+            if not isinstance(url, str) or not url:
+                return None
+            return {"type": "input_file", "file_url": url}  # mutable-ok: API message payload
+        return None
+
+    @staticmethod
+    def _tool_result_output_value(
+        output_text: str,
+        file_parts: tuple[dict[str, str], ...],  # mutable-ok: json content parts
+    ) -> str | list[dict[str, str]]:  # mutable-ok: API message payload
+        """Plain string output, or a part list when document file parts are present."""
+        if not file_parts:
+            return output_text
+        text_parts: Final = (
+            [{"type": "input_text", "text": output_text}] if output_text else []  # mutable-ok: API message payload
+        )
+        return [*text_parts, *file_parts]  # mutable-ok: API message payload
+
     @staticmethod
     def _translate_midturn_system_content_to_responses(
         content: str | Iterable[AnthropicSystemMessageContent],
@@ -169,6 +214,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
           system text        -> message(role=system, input_text)
           user text          -> message(role=user, input_text)
           user image         -> message(role=user, input_image)
+          user document      -> message(role=user, input_file)
           user tool_result   -> function_call_output
           assistant text     -> message(role=assistant, output_text)
           assistant thinking -> reasoning
@@ -223,9 +269,25 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                         {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
                                     )
                                 )
+                        elif btype == "document":
+                            file_part = self._translate_anthropic_document_block_to_file_part(block)
+                            if file_part:
+                                user_parts.append(
+                                    with_prompt_cache_breakpoint(file_part, block.get("prompt_cache_breakpoint"))
+                                )
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
+                            document_candidates = (
+                                tuple(
+                                    self._translate_anthropic_document_block_to_file_part(c)
+                                    for c in inner
+                                    if isinstance(c, dict) and c.get("type") == "document"
+                                )
+                                if isinstance(inner, list)
+                                else ()
+                            )
+                            tool_file_parts = tuple(part for part in document_candidates if part is not None)
                             if inner is None:
                                 output_text = ""
                             elif isinstance(inner, str):
@@ -258,7 +320,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                 {
                                     "type": "function_call_output",
                                     "call_id": tool_use_id,
-                                    "output": output_text,
+                                    "output": self._tool_result_output_value(output_text, tool_file_parts),
                                 }
                             )
                     if tool_image_parts:

litellm/llms/anthropic/files/handler.py

@@ -22,19 +22,7 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import CallTypes, LlmProviders, ModelResponse
 
 from ..chat.transformation import AnthropicConfig
-from ..common_utils import AnthropicModelInfo
-
-# Map Anthropic error types to HTTP status codes
-ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = {
-    "invalid_request_error": 400,
-    "authentication_error": 401,
-    "permission_error": 403,
-    "not_found_error": 404,
-    "rate_limit_error": 429,
-    "api_error": 500,
-    "overloaded_error": 503,
-    "timeout_error": 504,
-}
+from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo
 
 
 class AnthropicFilesHandler:

litellm/llms/azure/search/__init__.py

@@ -0,0 +1,3 @@
+from litellm.llms.azure.search.transformation import BingGroundingSearchConfig
+
+__all__ = ("BingGroundingSearchConfig",)

litellm/llms/azure/search/transformation.py

@@ -0,0 +1,442 @@
+"""
+Calls the Microsoft Foundry Responses API with the `bing_grounding` or `web_search`
+tool to search the web (Grounding with Bing Search).
+
+Microsoft docs: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding
+
+Setup:
+    1. Set BING_GROUNDING_PROJECT_ENDPOINT to the Foundry project endpoint, e.g.
+       https://<account>.services.ai.azure.com/api/projects/<project>
+    2. Set BING_GROUNDING_MODEL to a model deployment in that project (e.g. gpt-4.1);
+       it runs the grounded search and its tokens are billed on that deployment
+    3. Optional: set BING_GROUNDING_CONNECTION_ID to a Grounding with Bing Search
+       project connection id to use the `bing_grounding` tool; without it the
+       project's built-in `web_search` tool is used
+    4. Auth: pass api_key (an Azure API key, sent in the api-key header), or set
+       BING_GROUNDING_TOKEN to an Entra bearer token for scope
+       https://ai.azure.com/.default, or configure azure-identity (AZURE_CLIENT_ID /
+       AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, or any
+       DefaultAzureCredential source) and the token is minted automatically
+
+Usage:
+    response = litellm.search(
+        query="latest AI developments",
+        search_provider="bing_grounding",
+        max_results=5,
+    )
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+import httpx
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.search.transformation import (
+    BaseSearchConfig,
+    SearchResponse,
+    SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_DOCS_URL: Final = "https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding"
+
+PROJECT_ENDPOINT_ENV: Final = "BING_GROUNDING_PROJECT_ENDPOINT"
+MODEL_ENV: Final = "BING_GROUNDING_MODEL"
+CONNECTION_ID_ENV: Final = "BING_GROUNDING_CONNECTION_ID"
+TOKEN_ENV: Final = "BING_GROUNDING_TOKEN"
+
+ENTRA_SCOPE: Final = "https://ai.azure.com/.default"
+
+_RESPONSES_PATH: Final = "/openai/v1/responses"
+_SNIPPET_FALLBACK_LENGTH: Final = 300
+_UPSTREAM_ERROR_STATUS: Final = 502
+_RESPONSE_COST_HEADER: Final = "llm_provider-x-litellm-response-cost"
+
+
+class _Annotation(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    url: str | None = None
+    title: str | None = None
+    start_index: int | None = None
+    end_index: int | None = None
+
+
+class _ContentPart(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    text: str = ""
+    annotations: tuple[_Annotation, ...] = ()
+
+
+class _OutputItem(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    content: tuple[_ContentPart, ...] = ()
+
+
+class _ErrorBody(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    message: str | None = None
+
+
+class _IncompleteDetails(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    reason: str | None = None
+
+
+class _ResponsesEnvelope(BaseModel):
+    """A Foundry Responses API body. `output` is required: a body without it is not a
+    Responses API response and must not be reported as a successful empty search.
+
+    A 200 body can still carry `status` `failed` or `incomplete`; those are surfaced as
+    errors rather than reported as a successful empty search."""
+
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    output: tuple[_OutputItem, ...]
+    status: str | None = None
+    error: _ErrorBody | None = None
+    incomplete_details: _IncompleteDetails | None = None
+
+
+class _ErrorEnvelope(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    error: _ErrorBody | None = None
+
+
+def _unwrap_error_detail(error_message: str) -> str:
+    """
+    Surface the human-readable message inside Foundry's error envelope.
+
+    Tool failures nest a second JSON document as a string inside `error.message`
+    (observed live for `bing_grounding` connection errors), so the unwrap runs twice.
+    Falls back to the raw body for anything else.
+    """
+    try:
+        envelope: Final = _ErrorEnvelope.model_validate_json(error_message)
+    except ValidationError:
+        return error_message
+    message: Final = envelope.error.message if envelope.error else None
+    if message is None:
+        return error_message
+    try:
+        nested: Final = _ErrorBody.model_validate_json(message)
+    except ValidationError:
+        return message
+    return nested.message or message
+
+
+def _snippet(text: str, annotation: _Annotation) -> str:
+    """
+    The text a citation supports, not the citation marker itself.
+
+    A url_citation's start/end indices span the inline marker ("([host](url))"),
+    which follows the claim it backs, so the snippet is the marker's own line up
+    to where the marker starts.
+    """
+    start: Final = annotation.start_index
+    marker_start: Final = start if start is not None and 0 <= start <= len(text) else len(text)
+    claim: Final = text[:marker_start].rsplit("\n", 1)[-1].strip()
+    if claim:
+        return claim[-_SNIPPET_FALLBACK_LENGTH:]
+    return text[:_SNIPPET_FALLBACK_LENGTH]
+
+
+def _citation_results(envelope: _ResponsesEnvelope) -> tuple[SearchResult, ...]:
+    """One result per cited URL: first occurrence wins, order preserved as answered."""
+    cited: Final = tuple(
+        SearchResult(
+            title=annotation.title or "",
+            url=annotation.url or "",
+            snippet=_snippet(part.text, annotation),
+            date=None,
+            last_updated=None,
+        )
+        for item in envelope.output
+        if item.type == "message"
+        for part in item.content
+        if part.type == "output_text"
+        for annotation in part.annotations
+        if annotation.type == "url_citation" and annotation.url
+    )
+    first_by_url: Final = MappingProxyType({result.url: result for result in reversed(cited)})
+    return tuple(first_by_url[url] for url in dict.fromkeys(result.url for result in cited))
+
+
+def _valid_max_results(max_results: object) -> int | None:
+    """A positive-int `max_results`, else None. Rejects bools, an `int` subclass, and
+    non-positive values so neither the request-side `count` nor the response-side cap
+    forwards a value the other would silently ignore.
+    """
+    if isinstance(max_results, bool) or not isinstance(max_results, int):
+        return None
+    return max_results if max_results > 0 else None
+
+
+def _requested_max_results(response_kwargs: Mapping[str, object]) -> int | None:
+    """The unified `max_results` cap the caller asked for, if any.
+
+    The built-in web_search tool has no server-side result-count knob, so the cap is
+    enforced here after the fact; connection mode also honors it as a hard ceiling on
+    top of the tool's `count` hint.
+    """
+    optional_params: Final = response_kwargs.get("optional_params")
+    if not isinstance(optional_params, Mapping):
+        return None
+    return _valid_max_results(optional_params.get("max_results"))
+
+
+def _capped(results: tuple[SearchResult, ...], max_results: int | None) -> tuple[SearchResult, ...]:
+    return results[:max_results] if max_results is not None else results
+
+
+class _SearchConfiguration(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    project_connection_id: str
+    count: int | None = None
+
+
+class _BingGroundingParams(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    search_configurations: tuple[_SearchConfiguration, ...]
+
+
+class _BingGroundingTool(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["bing_grounding"] = "bing_grounding"
+    bing_grounding: _BingGroundingParams
+
+
+class _UserLocation(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["approximate"] = "approximate"
+    country: str
+
+
+class _WebSearchTool(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["web_search"] = "web_search"
+    user_location: _UserLocation | None = None
+
+
+class _ResponsesRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model: str
+    input: str
+    tools: tuple[_BingGroundingTool | _WebSearchTool, ...]
+
+
+def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | _WebSearchTool:
+    connection_id: Final = get_secret_str(CONNECTION_ID_ENV)
+    max_results: Final = optional_params.get("max_results")
+    country: Final = optional_params.get("country")
+    if connection_id:
+        configuration: Final = _SearchConfiguration(
+            project_connection_id=connection_id,
+            count=_valid_max_results(max_results),
+        )
+        return _BingGroundingTool(bing_grounding=_BingGroundingParams(search_configurations=(configuration,)))
+    location: Final = _UserLocation(country=country.upper()) if isinstance(country, str) else None
+    return _WebSearchTool(user_location=location)
+
+
+def _default_entra_token_minter() -> str:
+    from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
+
+    return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
+
+
+class BingGroundingSearchConfig(BaseSearchConfig):
+    def __init__(self, entra_token_minter: Callable[[], str] | None = None) -> None:
+        super().__init__()
+        self._entra_token_minter = entra_token_minter
+
+    @staticmethod
+    def ui_friendly_name() -> str:
+        return "Grounding with Bing Search"
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.validate_environment signature
+        api_key: str | None = None,
+        api_base: str | None = None,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.validate_environment signature
+    ) -> dict[str, str]:  # mutable-ok: the http handler passes this straight to httpx as headers
+        """
+        Validate environment and return headers.
+
+        Returns a new dict rather than mutating ``headers``: the http handler calls this
+        a second time after ``litellm/search/main.py`` already did, so it has to be idempotent.
+        """
+        return {  # mutable-ok: httpx requires a plain dict of headers
+            **headers,
+            **self._auth_header(api_key, api_base),
+            "Content-Type": "application/json",
+        }
+
+    def _auth_header(self, api_key: str | None, api_base: str | None) -> Mapping[str, str]:
+        """
+        A caller-supplied ``api_key`` is an Azure API key and rides the ``api-key`` header;
+        an Entra bearer token (``BING_GROUNDING_TOKEN`` or one minted via azure-identity)
+        rides ``Authorization: Bearer``. Foundry rejects the wrong scheme for each.
+        """
+        if api_key:
+            return MappingProxyType({"api-key": api_key})
+        token: Final = self.resolve_server_api_key(
+            caller_api_key=None,
+            caller_api_base=api_base,
+            key_env_vars=(TOKEN_ENV,),
+            base_env_var=PROJECT_ENDPOINT_ENV,
+            default_api_base=None,
+        ) or self._mint_entra_token(api_base)
+        return MappingProxyType({"Authorization": f"Bearer {token}"})
+
+    def _mint_entra_token(self, caller_api_base: str | None) -> str:
+        self._assert_trusted_api_base_for_server_credential(
+            caller_api_base, None, PROJECT_ENDPOINT_ENV, "Azure AD token"
+        )
+        minter: Final = self._entra_token_minter or _default_entra_token_minter
+        try:
+            return minter()
+        except Exception as e:
+            raise ValueError(
+                f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
+                f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "
+                f"AZURE_CLIENT_SECRET / AZURE_TENANT_ID or any DefaultAzureCredential source) "
+                f"for scope {ENTRA_SCOPE}. Underlying error: {e}"
+            ) from e
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        optional_params: dict[str, object],  # mutable-ok: BaseSearchConfig.get_complete_url signature
+        data: dict[str, object] | list[dict[str, object]] | None = None,  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.get_complete_url signature
+    ) -> str:
+        resolved_base: Final = api_base or get_secret_str(PROJECT_ENDPOINT_ENV)
+        if not resolved_base:
+            raise ValueError(
+                f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project "
+                f"endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>."
+            )
+        trimmed: Final = resolved_base.rstrip("/")
+        if trimmed.endswith(_RESPONSES_PATH):
+            return trimmed
+        return f"{trimmed}{_RESPONSES_PATH}"
+
+    def transform_search_request(
+        self,
+        query: str | list[str],  # mutable-ok: BaseSearchConfig.transform_search_request signature
+        optional_params: dict[str, object],  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_request signature
+    ) -> dict[str, object]:  # mutable-ok: the http handler passes this straight to httpx as the JSON body
+        """
+        Transform Search request to the Foundry Responses API format.
+
+        The unified params map as far as the API allows:
+        - max_results -> the bing_grounding search configuration's `count`; the built-in
+          web_search tool has no result-count knob, so that mode instead caps the returned
+          results after the fact (see transform_search_response)
+        - country -> web_search's approximate `user_location` (bing_grounding's `market`
+          wants a full locale like en-US, which a bare country code cannot fill)
+        - search_domain_filter, max_tokens_per_page -> no API equivalent, dropped
+        """
+        model: Final = get_secret_str(MODEL_ENV)
+        if not model:
+            raise ValueError(
+                f"{MODEL_ENV} is not set. Set it to a model deployment in the Foundry project "
+                f"that runs the grounded search, e.g. gpt-4.1."
+            )
+        request: Final = _ResponsesRequest(
+            model=model,
+            input=" ".join(query) if isinstance(query, list) else query,
+            tools=(_search_tool(optional_params),),
+        )
+        return request.model_dump(mode="json", exclude_none=True)
+
+    def transform_search_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_response signature
+    ) -> SearchResponse:
+        try:
+            parsed: Final = _ResponsesEnvelope.model_validate_json(raw_response.content)
+        except ValidationError as e:
+            raise self.get_error_class(
+                error_message=f"response does not match the Foundry Responses API schema: {e}",
+                status_code=raw_response.status_code,
+                headers=dict(raw_response.headers),  # mutable-ok: BaseSearchConfig.get_error_class signature
+            )
+        if parsed.status == "failed":
+            detail: Final = (
+                parsed.error.message if parsed.error and parsed.error.message else "the grounded search failed"
+            )
+            raise self._upstream_error(detail, raw_response)
+        results: Final = _capped(_citation_results(parsed), _requested_max_results(kwargs))
+        if not results and parsed.status == "incomplete":
+            reason: Final = (
+                parsed.incomplete_details.reason
+                if parsed.incomplete_details and parsed.incomplete_details.reason
+                else "unknown reason"
+            )
+            raise self._upstream_error(f"the grounded search was incomplete: {reason}", raw_response)
+        return self._priced(results)
+
+    def _upstream_error(self, detail: str, raw_response: httpx.Response) -> Exception:
+        return self.get_error_class(
+            error_message=detail,
+            status_code=_UPSTREAM_ERROR_STATUS,
+            headers=dict(raw_response.headers),  # mutable-ok: BaseSearchConfig.get_error_class signature
+        )
+
+    def _priced(self, results: tuple[SearchResult, ...]) -> SearchResponse:
+        """web_search mode runs no paid Grounding with Bing transaction, so it must not
+        inherit the connection-mode ``bing_grounding/search`` price; zero its per-query
+        cost while leaving connection mode to the cost map."""
+        response: Final = SearchResponse(
+            results=list(results),  # mutable-ok: SearchResponse.results is list[SearchResult]
+            object="search",
+        )
+        if get_secret_str(CONNECTION_ID_ENV):
+            return response
+        response._hidden_params[
+            "additional_headers"
+        ] = {  # mutable-ok: response_cost_calculator writes into _hidden_params
+            _RESPONSE_COST_HEADER: 0.0
+        }
+        return response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.get_error_class signature
+    ) -> Exception:
+        detail: Final = _unwrap_error_detail(error_message).rstrip(". ")
+        return BaseLLMException(
+            status_code=status_code,
+            message=f"Grounding with Bing Search: {detail}. See {_DOCS_URL} for details.",
+            headers=headers,
+        )

litellm/llms/azure_ai/azure_model_router/transformation.py

@@ -65,15 +65,24 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
 
         Extracts the actual model used from the Azure response (e.g., gpt-5-nano-2025-08-07)
         and returns it with the azure_ai/ prefix for proper display and cost tracking.
+
+        Also stamps that model onto ``_hidden_params`` so downstream consumers (spend logs,
+        response restamping) can read it instead of guessing the route from the model string.
         """
-        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+        from litellm.llms.azure_ai.common_utils import (
+            AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+            AzureFoundryModelInfo,
+        )
+        from litellm.router_utils.add_retry_fallback_headers import (
+            get_hidden_params_dict,
+        )
 
         # Get base model for the parent call (strips routing prefixes for API compatibility)
         base_model: Final[str] = AzureFoundryModelInfo.get_base_model(model)
 
         # Call parent transform_response first - this will extract the actual model
         # from the raw response (e.g., "gpt-5-nano-2025-08-07")
-        model_response = super().transform_response(
+        transformed_response: Final = super().transform_response(
             model=base_model,
             raw_response=raw_response,
             model_response=model_response,
@@ -86,7 +95,15 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
             api_key=api_key,
             json_mode=json_mode,
         )
-        return model_response
+        selected_model: Final = transformed_response.model
+        if selected_model:
+            # Rebuilt rather than mutated in place: ModelResponseBase declares _hidden_params as a
+            # class-level dict, so an in-place write can bleed into unrelated responses.
+            transformed_response._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # ModelResponse exposes no public hidden-params setter  # mutable-ok: ModelResponse requires _hidden_params to be a plain dict
+                **get_hidden_params_dict(transformed_response),
+                AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model,
+            }
+        return transformed_response
 
     def calculate_additional_costs(self, model: str, prompt_tokens: int, completion_tokens: int) -> dict | None:
         """

litellm/llms/azure_ai/common_utils.py

@@ -51,6 +51,9 @@ def get_azure_ai_auth_headers(
     )
 
 
+AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model"
+
+
 class AzureFoundryModelInfo(BaseLLMModelInfo):
     """Model info for Azure AI / Azure Foundry models."""
 
@@ -82,6 +85,41 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
             return "model_router"
         return "default"
 
+    @staticmethod
+    def get_model_router_selected_model(hidden_params: Mapping[str, object] | None) -> str | None:
+        """The model Azure Model Router actually served, stamped by ``AzureModelRouterConfig``.
+
+        Reading this beats re-deriving the route from a model string: the stamp is set on the
+        code path that was actually taken, so it holds no matter what the caller named the model.
+        """
+        if not hidden_params:
+            return None
+        selected: Final = hidden_params.get(AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY)
+        if isinstance(selected, str) and selected:
+            return selected
+        return None
+
+    @staticmethod
+    def is_model_router_call(
+        model: str | None = None,
+        hidden_params: Mapping[str, object] | None = None,
+    ) -> bool:
+        """Whether a request went down the Azure Model Router route.
+
+        Prefers the response stamp, then the deployment's litellm model path, and only then the
+        caller-supplied name. The last two go through ``get_azure_ai_route`` so the model-router
+        name heuristic lives in exactly one place.
+        """
+        if AzureFoundryModelInfo.get_model_router_selected_model(hidden_params) is not None:
+            return True
+        deployment_model: Final = (
+            hidden_params.get("litellm_model_name") or hidden_params.get("model") if hidden_params is not None else None
+        )
+        return any(
+            isinstance(candidate, str) and AzureFoundryModelInfo.get_azure_ai_route(candidate) == "model_router"
+            for candidate in (deployment_model, model)
+        )
+
     @staticmethod
     def get_api_base(api_base: str | None = None) -> str | None:
         return api_base or litellm.api_base or get_secret_str("AZURE_AI_API_BASE")

litellm/llms/base_llm/videos/transformation.py

@@ -1,9 +1,10 @@
 import types
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any
 
 import httpx
-from httpx._types import RequestFiles
+from httpx._types import FileContent, RequestFiles
 
 from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
@@ -340,14 +341,18 @@ class BaseVideoConfig(ABC):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, Any] | None = None,
         prefetched_source_data: dict[str, Any] | None = None,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         """
-        Transform the video edit request into a URL and JSON data.
+        Transform the video edit request into a URL plus either JSON data or
+        multipart form fields and files.
 
         Returns:
-            Tuple[str, Dict]: (url, data) for the POST request
+            tuple[str, Mapping[str, object], RequestFiles | None]: (url, data,
+            files). When files is None the handler sends data as JSON; otherwise
+            data holds the form fields and files holds the uploaded source video.
         """
         raise NotImplementedError("video edit is not supported for this provider")
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1617,6 +1617,8 @@ class AmazonConverseConfig(BaseConfig):
         }
         if additional_request_params:
             data["additionalModelRequestFields"] = additional_request_params
+            if "thinking" in additional_request_params:
+                data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",)
         if system_content_blocks:
             data["system"] = system_content_blocks
 
@@ -1801,6 +1803,17 @@ class AmazonConverseConfig(BaseConfig):
                 thinking_blocks_list.append(_redacted_block)
         return thinking_blocks_list
 
+    @staticmethod
+    def thinking_tokens_from_additional_fields(additional_fields: object) -> int | None:
+        """Converse omits thinking tokens from its usage block; they only arrive under
+        ``additionalModelResponseFields`` when ``/usage/output_tokens_details`` is requested."""
+        if not isinstance(additional_fields, Mapping):
+            return None
+        usage: Final = additional_fields.get("usage")
+        if not isinstance(usage, Mapping):
+            return None
+        return AnthropicConfig.thinking_tokens_from_usage(usage)
+
     @staticmethod
     def is_converse_usage_shape(usage_object: Mapping[str, object]) -> bool:
         """Converse-family models report camelCase token counts, not Anthropic's snake_case."""
@@ -1842,6 +1855,7 @@ class AmazonConverseConfig(BaseConfig):
         usage: ConverseTokenUsageBlock,
         reasoning_content: str | None = None,
         thinking_ran: bool = False,
+        provider_reasoning_tokens: int | None = None,
     ) -> Usage:
         input_tokens = usage["inputTokens"]
         output_tokens: Final = usage["outputTokens"]
@@ -1862,9 +1876,14 @@ class AmazonConverseConfig(BaseConfig):
             cache_creation_tokens=cache_creation_input_tokens,
             text_tokens=raw_input_tokens,
         )
-        reasoning_tokens: Final = (
+        estimated_reasoning_tokens: Final = (
             token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0
         )
+        reasoning_tokens: Final = (
+            min(max(0, provider_reasoning_tokens), output_tokens)
+            if provider_reasoning_tokens is not None
+            else estimated_reasoning_tokens
+        )
         completion_tokens_details: Final = (
             CompletionTokensDetailsWrapper(
                 reasoning_tokens=reasoning_tokens,
@@ -2272,6 +2291,9 @@ class AmazonConverseConfig(BaseConfig):
             completion_response["usage"],
             reasoning_content=chat_completion_message.get("reasoning_content"),
             thinking_ran=reasoningContentBlocks is not None,
+            provider_reasoning_tokens=self.thinking_tokens_from_additional_fields(
+                completion_response.get("additionalModelResponseFields")
+            ),
         )
 
         ## HANDLE TOOL CALLS

litellm/llms/bedrock/chat/invoke_handler.py

@@ -331,6 +331,7 @@ class AWSEventStreamDecoder:
         self.json_mode = json_mode
         self._current_tool_name: str | None = None
         self._thinking_ran = False
+        self._provider_reasoning_tokens: int | None = None
 
     def check_empty_tool_call_args(self) -> bool:
         """
@@ -559,10 +560,14 @@ class AWSEventStreamDecoder:
                 tool_use = self._handle_converse_stop_event(content_block_index)
             elif "stopReason" in chunk_data:
                 finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
+                self._provider_reasoning_tokens = AmazonConverseConfig.thinking_tokens_from_additional_fields(
+                    chunk_data.get("additionalModelResponseFields")
+                )
             elif "usage" in chunk_data:
                 usage = converse_config.transform_usage(
                     chunk_data.get("usage", {}),
                     thinking_ran=self._thinking_ran,
+                    provider_reasoning_tokens=self._provider_reasoning_tokens,
                 )
             if thinking_blocks:
                 self._thinking_ran = True

litellm/llms/bedrock/passthrough/transformation.py

@@ -1,4 +1,5 @@
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final, Optional, cast
 
 from httpx import Response
@@ -93,6 +94,9 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
             endpoint_url,
         )
 
+    def get_bedrock_bearer_token(self, litellm_params: Mapping[str, object]) -> str | None:
+        return None
+
     def sign_request(
         self,
         headers: dict,
@@ -109,6 +113,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
             request_data=request_data or {},
             api_base=api_base,
             model=model,
+            api_key=self.get_bedrock_bearer_token(optional_params),
         )
 
     def logging_non_streaming_response(

litellm/llms/bedrock/rerank/handler.py

@@ -29,6 +29,7 @@ class BedrockRerankHandler(BaseAWSLLM):
     async def arerank(
         self,
         prepared_request: BedrockPreparedRequest,
+        logging_obj: LitellmLogging,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ):
@@ -40,6 +41,7 @@ class BedrockRerankHandler(BaseAWSLLM):
                 headers=dict(prepared_request["prepped"].headers),
                 data=prepared_request["body"],
                 timeout=timeout,
+                logging_obj=logging_obj,
             )
             response.raise_for_status()
         except httpx.HTTPStatusError as err:
@@ -98,6 +100,7 @@ class BedrockRerankHandler(BaseAWSLLM):
         if _is_async:
             return self.arerank(
                 prepared_request,
+                logging_obj=logging_obj,
                 timeout=timeout,
                 client=client if client is not None and isinstance(client, AsyncHTTPHandler) else None,
             )

litellm/llms/bedrock_mantle/common_utils.py

@@ -13,6 +13,7 @@ global state.
 """
 
 import re
+from collections.abc import Mapping
 from typing import Final
 
 from botocore.exceptions import (
@@ -31,30 +32,39 @@ BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1"
 MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE)
 
 
+def resolve_mantle_bearer_token(api_key: str | None) -> str | None:
+    return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+
+
+def resolve_mantle_region(params: Mapping[str, object]) -> str:
+    region: Final = params.get("aws_region_name")
+    if isinstance(region, str) and region:
+        BaseAWSLLM._validate_aws_region_name(region)
+        return region
+    api_base: Final = params.get("api_base")
+    base: Final = (api_base if isinstance(api_base, str) else None) or get_secret_str("BEDROCK_MANTLE_API_BASE")
+    if base:
+        match: Final = MANTLE_HOST_RE.match(base.rstrip("/"))
+        if match:
+            return match.group(1)
+    return (
+        get_secret_str("BEDROCK_MANTLE_REGION")
+        or get_secret_str("AWS_REGION_NAME")
+        or get_secret_str("AWS_REGION")
+        or BEDROCK_MANTLE_DEFAULT_REGION
+    )
+
+
 class BedrockMantleAuthMixin:
     _aws_signer: BaseAWSLLM
 
     @staticmethod
     def _resolve_bearer_token(api_key: str | None) -> str | None:
-        return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+        return resolve_mantle_bearer_token(api_key)
 
     @staticmethod
     def _resolve_region(params: dict) -> str:
-        region: Final = params.get("aws_region_name")
-        if region:
-            BaseAWSLLM._validate_aws_region_name(region)
-            return region
-        base: Final = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
-        if base:
-            match: Final = MANTLE_HOST_RE.match(base.rstrip("/"))
-            if match:
-                return match.group(1)
-        return (
-            get_secret_str("BEDROCK_MANTLE_REGION")
-            or get_secret_str("AWS_REGION_NAME")
-            or get_secret_str("AWS_REGION")
-            or BEDROCK_MANTLE_DEFAULT_REGION
-        )
+        return resolve_mantle_region(params)
 
     def sign_request(
         self,

litellm/llms/bedrock_mantle/passthrough/transformation.py

@@ -0,0 +1,71 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final, Literal, Optional
+
+from httpx import Response
+
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig
+from litellm.llms.bedrock_mantle.common_utils import (
+    MANTLE_HOST_RE,
+    resolve_mantle_bearer_token,
+    resolve_mantle_region,
+)
+from litellm.types.utils import LlmProviders
+
+if TYPE_CHECKING:
+    from litellm.types.utils import CostResponseTypes
+
+
+class BedrockMantlePassthroughConfig(BedrockPassthroughConfig):
+    """Native Bedrock runtime passthrough (InvokeModel, Converse) for deployments declared as bedrock_mantle.
+
+    The Mantle host only serves the OpenAI-compatible surface, so a Mantle api_base lends its region and the
+    request itself goes to bedrock-runtime, signed with the deployment's Bearer token or SigV4 credentials.
+    """
+
+    def _get_aws_region_name(
+        self,
+        optional_params: Mapping[str, object],
+        model: str | None = None,
+        model_id: str | None = None,
+    ) -> str:
+        return resolve_mantle_region(optional_params)
+
+    def get_runtime_endpoint(
+        self,
+        api_base: str | None,
+        aws_bedrock_runtime_endpoint: str | None,
+        aws_region_name: str,
+        endpoint_type: Literal["runtime", "agent", "agentcore"] | None = "runtime",
+    ) -> tuple[str, str]:
+        is_mantle_host: Final = api_base is not None and MANTLE_HOST_RE.match(api_base.rstrip("/")) is not None
+        return super().get_runtime_endpoint(
+            api_base=None if is_mantle_host else api_base,
+            aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+            aws_region_name=aws_region_name,
+            endpoint_type=endpoint_type,
+        )
+
+    def get_bedrock_bearer_token(self, litellm_params: Mapping[str, object]) -> str | None:
+        api_key: Final = litellm_params.get("api_key")
+        return resolve_mantle_bearer_token(api_key if isinstance(api_key, str) else None)
+
+    def logging_non_streaming_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        httpx_response: Response,
+        request_data: dict,  # mutable-ok: mirrors the inherited BedrockPassthroughConfig signature
+        logging_obj: Logging,
+        endpoint: str,
+    ) -> Optional["CostResponseTypes"]:
+        is_converse: Final = "invoke" not in endpoint and "converse" in endpoint
+        shape_provider: Final = LlmProviders.BEDROCK.value if is_converse else custom_llm_provider
+        return super().logging_non_streaming_response(
+            model=model,
+            custom_llm_provider=shape_provider,
+            httpx_response=httpx_response,
+            request_data=request_data,
+            logging_obj=logging_obj,
+            endpoint=endpoint,
+        )

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -15,8 +15,12 @@ role / access key / profile / web identity), signed via the shared
 BaseAWSLLM._sign_request after the request body is finalized.
 """
 
+import json
+from collections.abc import Mapping
 from typing import Any, Final
 
+from typing_extensions import ReadOnly, TypedDict
+
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@@ -50,6 +54,33 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 
 _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
 
+_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
+_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
+_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
+
+
+class _RewrittenOutputTextBlock(TypedDict):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+
+
+class _RewrittenAssistantMessageItem(TypedDict):
+    type: ReadOnly[str]
+    role: ReadOnly[str]
+    content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
+
+
+class _RewrittenCompactionItem(TypedDict):
+    type: ReadOnly[str]
+    encrypted_content: ReadOnly[str]
+
+
+class _RewrittenFunctionCallItem(TypedDict):
+    type: ReadOnly[str]
+    call_id: ReadOnly[str]
+    name: ReadOnly[str]
+    arguments: ReadOnly[str]
+
 
 class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
     def __init__(
@@ -155,6 +186,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         headers: dict,
     ) -> dict:
         remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
+        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
         request_params: Final = (
             {
                 **response_api_optional_request_params,
@@ -168,7 +200,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return super().transform_responses_api_request(
             model=model,
-            input=remaining_input,
+            input=normalized_input,
             response_api_optional_request_params=request_params,
             litellm_params=litellm_params,
             headers=headers,
@@ -210,6 +242,91 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
 
+    @staticmethod
+    def _agent_message_text(item: "Mapping[str, Any]") -> str:
+        content: Final = item.get("content")
+        if not isinstance(content, list):
+            return ""
+        return "".join(
+            str(block.get("text") or block.get("encrypted_content") or "")
+            for block in content
+            if isinstance(block, dict)
+        )
+
+    @classmethod
+    def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
+        text: Final = cls._agent_message_text(item)
+        if not text:
+            return None
+        rewritten: Final[_RewrittenAssistantMessageItem] = {
+            "type": "message",
+            "role": "assistant",
+            "content": ({"type": "output_text", "text": text},),
+        }
+        return rewritten
+
+    @staticmethod
+    def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
+        encrypted_content: Final = item.get("encrypted_content")
+        if not isinstance(encrypted_content, str) or not encrypted_content:
+            return None
+        rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
+        return rewritten
+
+    @staticmethod
+    def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
+        call_id: Final = item.get("call_id")
+        if not isinstance(call_id, str) or not call_id:
+            return None
+        action: Final = item.get("action")
+        rewritten: Final[_RewrittenFunctionCallItem] = {
+            "type": "function_call",
+            "call_id": call_id,
+            "name": "local_shell",
+            "arguments": json.dumps(action) if isinstance(action, dict) else "{}",
+        }
+        return rewritten
+
+    @classmethod
+    def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]":
+        """Returns (normalized item or None to drop it, original type when rewritten)."""
+        if not isinstance(item, dict):
+            return item, None
+        item_type: Final = item.get("type")
+        if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE:
+            return cls._normalize_agent_message_item(item), item_type
+        if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
+            return cls._normalize_context_compaction_item(item), item_type
+        if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
+            return cls._normalize_local_shell_call_item(item), item_type
+        return item, None
+
+    @classmethod
+    def _normalize_codex_input_items(
+        cls,
+        input: "str | ResponseInputParam",
+    ) -> "str | ResponseInputParam":
+        """Rewrite Codex history item types Mantle rejects with 400 "Invalid
+        'input': value did not match any expected variant" into supported
+        equivalents. `agent_message` (Codex multi-agent traffic; its
+        encrypted_content slot carries the plaintext payload when the model
+        never issued encrypted args) becomes an assistant message,
+        `context_compaction` becomes the `compaction` spelling Mantle accepts,
+        and `local_shell_call` becomes the function_call its recorded
+        function_call_output already pairs with.
+        """
+        if not isinstance(input, list):
+            return input
+        normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input)
+        rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))
+        if rewritten_types:
+            verbose_logger.warning(
+                "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
+                rewritten_types,
+            )
+        kept: Final = [item for item, _ in normalized if item is not None]  # mutable-ok: ResponseInputParam is a list
+        return kept  # pyright: ignore[reportReturnType]  # Codex passthrough items sit outside the OpenAI input union
+
     def map_openai_params(
         self,
         response_api_optional_params: ResponsesAPIOptionalRequestParams,

litellm/llms/cerebras/chat.py

@@ -68,6 +68,8 @@ class CerebrasConfig(OpenAIGPTConfig):
             "tool_choice",
             "tools",
             "user",
+            "max_retries",
+            "extra_headers",
         ]
 
         # Only add reasoning_effort for models that support it

litellm/llms/custom_httpx/http_handler.py

@@ -9,7 +9,7 @@ import threading
 import time
 from collections.abc import AsyncIterable, Callable, Iterable, Mapping
 from http.cookiejar import CookieJar, DefaultCookiePolicy
-from typing import TYPE_CHECKING, Any, Final, Optional, TypeAlias, TypedDict
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, TypeAlias, TypedDict
 
 import certifi
 import httpx
@@ -933,11 +933,83 @@ class AsyncHTTPHandler:
         response.raise_for_status()
         return response
 
+    # Strong references to finalizer-scheduled client-close tasks. A bare
+    # create_task() result may be garbage-collected before it runs, leaving
+    # the underlying aiohttp session unclosed ("Unclosed client session").
+    # Mirrors LiteLLMAiohttpTransport._background_close_tasks.
+    _finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set()  # mutable-ok: strong refs for pending closes
+
+    @classmethod
+    def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
+        cls._finalizer_close_tasks.discard(task)
+        if task.cancelled():
+            return
+        exc: Final = task.exception()
+        if exc is not None:
+            verbose_logger.debug("Error closing client at finalization: %s", exc)
+
+    def _aiohttp_session_bound_elsewhere(self, loop: asyncio.AbstractEventLoop) -> bool:
+        """True when the wrapped aiohttp session is bound to a loop other than
+        ``loop`` — awaiting ``aclose()`` here would touch that loop's internals."""
+        from litellm.llms.custom_httpx.aiohttp_transport import (
+            LiteLLMAiohttpTransport,
+        )
+
+        transport: Final = getattr(self._client, "_transport", None)
+        if not isinstance(transport, LiteLLMAiohttpTransport):
+            return False
+        session: Final = transport.client
+        if not isinstance(session, ClientSession) or session.closed:
+            return False
+        return getattr(session, "_loop", None) is not loop
+
+    def _dispose_wrapped_aiohttp_session(self) -> None:
+        """Dispose the wrapped aiohttp session when ``aclose()`` cannot run here.
+
+        Finalization either has no running loop, or a loop the session is not
+        bound to. Delegating to the transport's lifecycle-aware disposal picks
+        the safe path per session state (async close on its own loop, threadsafe
+        handoff to a loop running elsewhere, or the synchronous connector
+        teardown that flips the flags ``ClientSession.__del__`` checks), so no
+        "Unclosed client session" / "Unclosed connector" warnings fire at
+        garbage collection.
+        """
+        from litellm.llms.custom_httpx.aiohttp_transport import (
+            LiteLLMAiohttpTransport,
+        )
+
+        transport: Final = getattr(self._client, "_transport", None)
+        if not isinstance(transport, LiteLLMAiohttpTransport):
+            return
+        # A shared session (e.g. the proxy's) is never this handler's to close.
+        if not getattr(transport, "_owns_session", False):
+            return
+        session: Final = transport.client
+        if isinstance(session, ClientSession) and not session.closed:
+            transport._close_recycled_session(session)  # pyright: ignore[reportPrivateUsage]  # deliberate reuse of the transport's lifecycle-aware disposal; an async close can never run in this context
+
     def __del__(self) -> None:
         try:
             if not _handler_may_close_client(sys.getrefcount(self._client), self._owns_client):
                 return
-            asyncio.get_running_loop().create_task(self._client.aclose())
+            try:
+                loop: Final = asyncio.get_running_loop()
+            except RuntimeError:
+                # No running loop at finalization time (worker threads after
+                # their loop closed, interpreter/worker shutdown, GC in a
+                # sync context). An async close can never run here.
+                self._dispose_wrapped_aiohttp_session()
+                return
+            if self._aiohttp_session_bound_elsewhere(loop):
+                # GC ran on a live loop (e.g. the app's) but the session
+                # belongs to another, possibly dead, loop — awaiting aclose()
+                # here is the cross-loop path the transport refuses.
+                self._dispose_wrapped_aiohttp_session()
+                return
+            task: Final = loop.create_task(self._client.aclose())
+            cls: Final = type(self)
+            cls._finalizer_close_tasks.add(task)
+            task.add_done_callback(cls._on_finalizer_close_done)
         except Exception:
             pass
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -24,6 +24,7 @@ from typing import (
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import httpx
+from httpx._types import FileContent
 from openai.types.file_deleted import FileDeleted
 
 import litellm
@@ -1265,6 +1266,7 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 data=json.dumps(request_data),
                 timeout=timeout,
+                logging_obj=logging_obj,
             )
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
@@ -1909,6 +1911,7 @@ class BaseLLMHTTPHandler:
         return provider_config.transform_search_response(
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def async_search(
@@ -2007,6 +2010,7 @@ class BaseLLMHTTPHandler:
         return provider_config.transform_search_response(
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def _async_post_anthropic_messages_with_http_error_retry(
@@ -2327,6 +2331,10 @@ class BaseLLMHTTPHandler:
                 AgenticAnthropicStreamingIterator,
             )
 
+            held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
+                logging_obj=logging_obj,
+                tools=anthropic_messages_optional_request_params.get("tools"),
+            )
             initial_response = AgenticAnthropicStreamingIterator(
                 completion_stream=completion_stream,
                 http_handler=self,
@@ -2337,6 +2345,8 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
+                hold_back=bool(held_back_tool_names),
+                server_fulfilled_tool_names=held_back_tool_names,
             )
             return AnthropicMessagesStreamingResponse(
                 completion_stream=initial_response,
@@ -5188,6 +5198,20 @@ class BaseLLMHTTPHandler:
                 return True
         return False
 
+    @staticmethod
+    def _server_fulfilled_tools_in_request(logging_obj: LiteLLMLoggingObj, tools: object) -> frozenset[str]:
+        """The request's tools that a registered callback fulfills server-side (e.g. ``headroom_retrieve``)."""
+        if not isinstance(tools, list) or not tools:
+            return frozenset()
+        from litellm.litellm_core_utils.prompt_templates.factory import has_tool_with_name
+
+        return frozenset(
+            name
+            for cb in _custom_logger_callbacks(logging_obj)
+            for name in getattr(cb, "server_fulfilled_tool_names", frozenset())
+            if has_tool_with_name(tools, name)
+        )
+
     @staticmethod
     def _check_agentic_loop_safety(
         tool_calls: object,
@@ -7905,6 +7929,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str,
         litellm_params,
         logging_obj,
+        video_file: FileContent | None = None,
         extra_headers: dict[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
         timeout: float | None = None,
@@ -7916,6 +7941,7 @@ class BaseLLMHTTPHandler:
             return self.async_video_edit_handler(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 video_provider_config=video_provider_config,
                 custom_llm_provider=custom_llm_provider,
                 litellm_params=litellm_params,
@@ -7969,9 +7995,10 @@ class BaseLLMHTTPHandler:
             prefetched_source_data = prefetch_resp.json()
 
         try:
-            url, data = video_provider_config.transform_video_edit_request(
+            url, data, files = video_provider_config.transform_video_edit_request(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 headers=headers,
@@ -7990,11 +8017,10 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            response: Final = sync_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=data,
-                timeout=timeout,
+            response: Final = (
+                sync_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
+                if files
+                else sync_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
             )
             response.raise_for_status()
             return video_provider_config.transform_video_edit_response(
@@ -8014,6 +8040,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str,
         litellm_params,
         logging_obj,
+        video_file: FileContent | None = None,
         extra_headers: dict[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
         timeout: float | None = None,
@@ -8065,9 +8092,10 @@ class BaseLLMHTTPHandler:
             prefetched_source_data = prefetch_resp.json()
 
         try:
-            url, data = video_provider_config.transform_video_edit_request(
+            url, data, files = video_provider_config.transform_video_edit_request(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 headers=headers,
@@ -8086,11 +8114,10 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            response: Final = await async_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=data,
-                timeout=timeout,
+            response: Final = await (
+                async_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
+                if files
+                else async_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
             )
             response.raise_for_status()
             return video_provider_config.transform_video_edit_response(

litellm/llms/gemini/videos/transformation.py

@@ -566,6 +566,7 @@ class GeminiVideoConfig(BaseVideoConfig):
         api_base,
         litellm_params,
         headers,
+        video_file=None,
         extra_body=None,
         prefetched_source_data=None,
     ):

litellm/llms/openai/chat/gpt_5_transformation.py

@@ -16,7 +16,7 @@ def _normalize_reasoning_effort_for_chat_completion(
 ) -> str | None:
     """Convert reasoning_effort to the string format expected by OpenAI chat completion API.
 
-    The chat completion API expects a simple string: 'none', 'low', 'medium', 'high', or 'xhigh'.
+    The chat completion API expects an effort string such as 'low' or 'high'.
     Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}.
     """
     if value is None:

litellm/llms/openai/videos/transformation.py

@@ -1,5 +1,7 @@
 import mimetypes
+from collections.abc import Mapping
 from io import BufferedReader, BytesIO
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, cast
 from urllib.parse import quote
 
@@ -502,15 +504,26 @@ class OpenAIVideoConfig(BaseVideoConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, object] | None = None,
         prefetched_source_data: dict[str, object] | None = None,
-    ) -> tuple[str, dict]:
-        original_video_id: Final = extract_original_video_id(video_id)
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         url: Final = f"{api_base.rstrip('/')}/edits"
+
+        if video_file is not None:
+            files: Final[RequestFiles] = (self._video_file_tuple(video_file, "video"),)
+            form_data: Final = (
+                MappingProxyType({"prompt": prompt, **extra_body})
+                if extra_body
+                else MappingProxyType({"prompt": prompt})
+            )
+            return url, form_data, files
+
+        original_video_id: Final = extract_original_video_id(video_id)
         data: Final[dict[str, object]] = {"prompt": prompt, "video": {"id": original_video_id}}
         if extra_body:
             data.update(extra_body)
-        return url, data
+        return url, data, None
 
     def transform_video_edit_response(
         self,
@@ -570,21 +583,22 @@ class OpenAIVideoConfig(BaseVideoConfig):
         else:
             files_list.append((field_name, ("input_reference.png", image, image_content_type)))
 
+    def _video_file_tuple(self, video: FileContent, field_name: str) -> tuple[str, FileTypes]:
+        """
+        Build a multipart field tuple for a video upload with proper video MIME
+        type detection: these paths must send video/mp4, not image/* content types.
+        """
+        filename: Final = getattr(video, "name", None) or "input_video.mp4"
+        content_type: Final = self._get_video_content_type(video=video, filename=filename)
+        return (field_name, (filename, video, content_type))
+
     def _add_video_to_files(
         self,
         files_list: list[tuple[str, FileTypes]],
         video: FileContent,
         field_name: str,
     ) -> None:
-        """
-        Add a video to files with proper video MIME type detection.
-
-        This path is used by POST /videos/characters and must send video/mp4,
-        not image/* content types.
-        """
-        filename: Final = getattr(video, "name", None) or "input_video.mp4"
-        content_type: Final = self._get_video_content_type(video=video, filename=filename)
-        files_list.append((field_name, (filename, video, content_type)))
+        files_list.append(self._video_file_tuple(video, field_name))
 
     def _get_video_content_type(self, video: FileContent, filename: str) -> str:
         guessed_content_type, _ = mimetypes.guess_type(filename)

litellm/llms/runwayml/videos/transformation.py

@@ -672,6 +672,7 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         api_base,
         litellm_params,
         headers,
+        video_file=None,
         extra_body=None,
         prefetched_source_data=None,
     ):

litellm/llms/together_ai/chat.py

@@ -1,58 +0,0 @@
-"""
-Support for OpenAI's `/v1/chat/completions` endpoint.
-
-Calls done in OpenAI/openai.py as TogetherAI is openai-compatible.
-
-Docs: https://docs.together.ai/reference/completions-1
-"""
-
-from typing import Final
-
-from litellm._logging import verbose_logger
-from litellm.utils import supports_function_calling
-
-from ..openai.chat.gpt_transformation import OpenAIGPTConfig
-
-
-class TogetherAIConfig(OpenAIGPTConfig):
-    def get_supported_openai_params(self, model: str) -> list:
-        """
-        Only some together models support response_format / tool calling
-
-        Docs: https://docs.together.ai/docs/json-mode
-        """
-        # Use supports_function_calling() — which reads _get_model_info_helper
-        # directly — instead of get_model_info(). get_model_info() calls
-        # get_supported_openai_params() as its first step, which routes back
-        # into this method for together_ai models, creating a recursion that
-        # only terminates when Python's recursion limit or the "not mapped"
-        # exception in _get_model_info_helper is hit (~332 deep calls).
-        supports_fc: bool | None = None
-        try:
-            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
-        except Exception as e:
-            verbose_logger.debug("Error getting supported openai params: %s", e)
-
-        optional_params: Final = super().get_supported_openai_params(model)
-        if supports_fc is not True:
-            verbose_logger.debug(
-                "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
-            )
-            optional_params.remove("tools")
-            optional_params.remove("tool_choice")
-            optional_params.remove("function_call")
-            optional_params.remove("response_format")
-        return optional_params
-
-    def map_openai_params(
-        self,
-        non_default_params: dict,
-        optional_params: dict,
-        model: str,
-        drop_params: bool,
-    ) -> dict:
-        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
-
-        if "response_format" in mapped_openai_params and mapped_openai_params["response_format"] == {"type": "text"}:
-            mapped_openai_params.pop("response_format")
-        return mapped_openai_params

litellm/llms/together_ai/chat/__init__.py

@@ -0,0 +1,3 @@
+from .transformation import TogetherAIChatConfig as TogetherAIChatConfig
+
+TogetherAIConfig = TogetherAIChatConfig

litellm/llms/together_ai/chat/transformation.py

@@ -0,0 +1,139 @@
+"""
+Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/completions`.
+
+Docs: https://docs.together.ai/docs/chat-overview
+"""
+
+from collections.abc import Container, Coroutine
+from types import MappingProxyType
+from typing import (
+    Final,
+    Literal,
+    cast,  # noqa: TID251  # rebuilding a TypedDict minus keys has no checked spelling
+    overload,
+)
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
+from litellm.types.llms.openai import AllMessageValues
+from litellm.utils import supports_function_calling
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
+LITELLM_INTERNAL_ASSISTANT_FIELDS: Final = frozenset({"thinking_blocks", "provider_specific_fields"})
+PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
+FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
+
+
+def _function_calling_verdict(model: str) -> bool | None:
+    try:
+        if supports_function_calling(model, custom_llm_provider="together_ai"):
+            return True
+    except Exception as e:
+        verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
+    registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
+    if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
+        return False
+    return None
+
+
+def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
+    passed_tool_params: Final = tuple(param for param in TOOL_CALLING_PARAMS if param in passed_params)
+    if not passed_tool_params:
+        return ()
+    verdict: Final = _function_calling_verdict(model)
+    if verdict is True:
+        return ()
+    if verdict is None:
+        verbose_logger.warning(
+            "together_ai model %s has no function calling entry in the model registry; passing %s through for Together to validate. Docs - %s",
+            model,
+            ", ".join(passed_tool_params),
+            FUNCTION_CALLING_DOCS_URL,
+        )
+        return ()
+    if drop_params or litellm.drop_params:
+        verbose_logger.warning(
+            "together_ai model %s does not support function calling per the model registry; dropping %s. Docs - %s",
+            model,
+            ", ".join(passed_tool_params),
+            FUNCTION_CALLING_DOCS_URL,
+        )
+        return passed_tool_params
+    raise UnsupportedParamsError(
+        status_code=500,
+        message=f"together_ai does not support parameters: {', '.join(passed_tool_params)}, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
+    )
+
+
+def _without_litellm_internal_fields(message: AllMessageValues) -> AllMessageValues:
+    if message["role"] != "assistant" or LITELLM_INTERNAL_ASSISTANT_FIELDS.isdisjoint(message):
+        return message
+    return cast(  # cast-ok: rebuilding the same TypedDict minus internal keys loses the narrowed type
+        "AllMessageValues",
+        {  # mutable-ok: TypedDict rebuild minus internal keys
+            key: value for key, value in message.items() if key not in LITELLM_INTERNAL_ASSISTANT_FIELDS
+        },
+    )
+
+
+class TogetherAIChatConfig(OpenAIGPTConfig):
+    @overload
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        model: str,
+        is_async: Literal[True],
+    ) -> Coroutine[object, object, list[AllMessageValues]]: ...  # mutable-ok: inherited contract
+
+    @overload
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        model: str,
+        is_async: Literal[False] = False,
+    ) -> list[AllMessageValues]: ...  # mutable-ok: inherited contract
+
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        model: str,
+        is_async: bool = False,
+    ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:  # mutable-ok: inherited contract
+        """Together consumes replayed assistant `reasoning_content` (preserved thinking via
+        `chat_template_kwargs: {"clear_thinking": false}`), so it must stay in the payload;
+        only litellm-internal fields are stripped before sending."""
+        stripped: Final = [  # mutable-ok: super() requires a list
+            _without_litellm_internal_fields(message) for message in messages
+        ]
+        if is_async:
+            return super()._transform_messages(stripped, model, is_async=True)
+        return super()._transform_messages(stripped, model, is_async=False)
+
+    def get_supported_openai_params(self, model: str) -> list:
+        supports_fc: Final = _function_calling_verdict(model)
+        supported_params: Final = super().get_supported_openai_params(model)
+        if supports_fc is True:
+            return supported_params
+        verbose_logger.debug(
+            "Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
+        )
+        return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
+            param for param in supported_params if param != "response_format"
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+        for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
+            mapped_openai_params.pop(param)
+        if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
+            mapped_openai_params.pop("response_format")
+        return mapped_openai_params

litellm/llms/together_ai/rerank/handler.py

@@ -16,11 +16,16 @@ from litellm.llms.together_ai.rerank.transformation import TogetherAIRerankConfi
 from litellm.types.rerank import RerankRequest, RerankResponse
 
 
+def _rerank_url(api_base: str) -> str:
+    return f"{api_base.rstrip('/')}/rerank"
+
+
 class TogetherAIRerank(BaseLLM):
     def rerank(
         self,
         model: str,
         api_key: str,
+        api_base: str,
         query: str,
         documents: list[str | dict[str, Any]],
         top_n: int | None = None,
@@ -46,10 +51,10 @@ class TogetherAIRerank(BaseLLM):
             raise ValueError("TogetherAI does not support max_chunks_per_doc")
 
         if _is_async:
-            return self.async_rerank(request_data_dict, api_key)  # Call async method
+            return self.async_rerank(request_data_dict, api_key, api_base)
 
         response: Final = client.post(
-            "https://api.together.xyz/v1/rerank",
+            _rerank_url(api_base),
             headers={
                 "accept": "application/json",
                 "content-type": "application/json",
@@ -69,11 +74,12 @@ class TogetherAIRerank(BaseLLM):
         self,
         request_data_dict: dict[str, Any],
         api_key: str,
+        api_base: str,
     ) -> RerankResponse:
         client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.TOGETHER_AI)  # Use async client
 
         response: Final = await client.post(
-            "https://api.together.xyz/v1/rerank",
+            _rerank_url(api_base),
             headers={
                 "accept": "application/json",
                 "content-type": "application/json",

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -23,6 +23,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE,
     DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
 )
+from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
 from litellm.litellm_core_utils.prompt_templates.factory import (
     _encode_tool_call_id_with_signature,
 )
@@ -3087,7 +3088,7 @@ class ModelResponseIterator:
         self.streaming_response = streaming_response
         self.response = response
         self.chunk_type: Literal["valid_json", "accumulated_json"] = "valid_json"
-        self.accumulated_json = ""
+        self._json_buffer = JSONFragmentAccumulator()
         self.sent_first_chunk = False
         self.logging_obj = logging_obj
         self.response_headers = response_headers or {}
@@ -3095,6 +3096,14 @@ class ModelResponseIterator:
         self.cumulative_tool_call_index: int = 0
         self.has_seen_tool_calls: bool = False
 
+    @property
+    def accumulated_json(self) -> str:
+        return self._json_buffer.snapshot()
+
+    @accumulated_json.setter
+    def accumulated_json(self, value: str) -> None:
+        self._json_buffer.set(value)
+
     @staticmethod
     def _check_streaming_error(chunk: dict) -> None:
         """Detect embedded errors (e.g. 429 RESOURCE_EXHAUSTED) in streaming chunks and raise VertexAIError."""
@@ -3298,8 +3307,8 @@ class ModelResponseIterator:
         return self.chunk_parser(chunk=json_chunk)
 
     def handle_accumulated_json_chunk(self, chunk: str, is_final: bool = False) -> Optional["ModelResponseStream"]:
-        message: Final = litellm.CustomStreamWrapper._strip_sse_data_from_chunk(chunk) or ""
-        self.accumulated_json = (self.accumulated_json + message.replace("\n\n", "")).strip()
+        message: Final = (litellm.CustomStreamWrapper._strip_sse_data_from_chunk(chunk) or "").replace("\n\n", "")
+        self._json_buffer.append(message)
 
         # Mid-stream, defer parsing until the buffer's last byte can close a value:
         # attempting a parse after every fragment of one large object is O(n^2) and
@@ -3307,27 +3316,23 @@ class ModelResponseIterator:
         # data is coming, so drain whatever complete values remain regardless of the
         # trailing byte, otherwise a complete leading value sitting behind a truncated
         # trailing one would be silently dropped.
-        if not is_final and (not self.accumulated_json or self.accumulated_json[-1] not in "}]"):
+        if not is_final and not self._json_buffer.could_close_json():
             return None
 
         # Peel one complete JSON value from the front of the buffer and keep the
         # unconsumed tail. Running json.loads over the whole buffer would fail
         # forever once it held more than one concatenated value ("Extra data") while
         # never resetting the buffer, so the buffer grew without bound and pinned the
-        # core. raw_decode reports where the value ended, so concatenated values drain
-        # one call at a time. A leading non-dict value (never emitted by Gemini in
-        # practice) is consumed and skipped so it cannot block the dict values behind it.
-        decoder: Final = json.JSONDecoder()
-        while self.accumulated_json:
-            try:
-                raw_value = decoder.raw_decode(self.accumulated_json)
-            except json.JSONDecodeError:
+        # core. pop_next_value reports where the value ended, so concatenated values
+        # drain one call at a time. A leading non-dict value (never emitted by Gemini
+        # in practice) is consumed and skipped so it cannot block the dict values
+        # behind it.
+        while True:
+            found, decoded = self._json_buffer.pop_next_value()
+            if not found:
                 return None
-            decoded, end_index = cast("tuple[object, int]", raw_value)  # cast-ok: raw_decode -> tuple[Any,int]
-            self.accumulated_json = self.accumulated_json[end_index:].strip()
             if isinstance(decoded, dict):
                 return self.chunk_parser(chunk=decoded)
-        return None
 
     def _common_chunk_parsing_logic(self, chunk: str) -> Optional["ModelResponseStream"]:
         try:
@@ -3351,7 +3356,7 @@ class ModelResponseIterator:
         try:
             chunk: Final = self.response_iterator.__next__()
         except StopIteration:
-            if self.chunk_type == "accumulated_json" and self.accumulated_json:
+            if self.chunk_type == "accumulated_json" and self._json_buffer:
                 result: Final = self.handle_accumulated_json_chunk(chunk="", is_final=True)
                 if result is not None:
                     return result
@@ -3375,7 +3380,7 @@ class ModelResponseIterator:
         try:
             chunk: Final = await self.async_response_iterator.__anext__()
         except StopAsyncIteration:
-            if self.chunk_type == "accumulated_json" and self.accumulated_json:
+            if self.chunk_type == "accumulated_json" and self._json_buffer:
                 result: Final = self.handle_accumulated_json_chunk(chunk="", is_final=True)
                 if result is not None:
                     return result

litellm/llms/vertex_ai/interactions/transformation.py

@@ -0,0 +1,149 @@
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Final
+
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
+from litellm.llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig
+from litellm.llms.vertex_ai.common_utils import validate_vertex_location
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+VERTEX_INTERACTIONS_API_VERSION: Final = "v1beta1"
+VERTEX_INTERACTIONS_DEFAULT_LOCATION: Final = "global"
+
+
+@dataclass(frozen=True, slots=True)
+class VertexInteractionsTarget:
+    base_url: str
+    project_id: str
+    location: str
+
+    @property
+    def collection_url(self) -> str:
+        return (
+            f"{self.base_url}/{VERTEX_INTERACTIONS_API_VERSION}"
+            f"/projects/{self.project_id}/locations/{self.location}/interactions"
+        )
+
+    def interaction_url(self, interaction_id: str) -> str:
+        encoded_interaction_id: Final = encode_url_path_segment(interaction_id, field_name="interaction_id")
+        return f"{self.collection_url}/{encoded_interaction_id}"
+
+
+class VertexAIInteractionsConfig(VertexBase, GoogleAIStudioInteractionsConfig):
+    def __init__(
+        self,
+        mint_access_token: Callable[[VERTEX_CREDENTIALS_TYPES | None, str | None], tuple[str, str]] | None = None,
+    ) -> None:
+        super().__init__()
+        self._mint_access_token: Final[Callable[[VERTEX_CREDENTIALS_TYPES | None, str | None], tuple[str, str]]] = (
+            mint_access_token or self._mint_access_token_with_vertex_base
+        )
+
+    def _mint_access_token_with_vertex_base(
+        self,
+        credentials: VERTEX_CREDENTIALS_TYPES | None,
+        project_id: str | None,
+    ) -> tuple[str, str]:
+        return self._ensure_access_token(
+            credentials=credentials, project_id=project_id, custom_llm_provider="vertex_ai"
+        )
+
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.VERTEX_AI
+
+    @property
+    def api_version(self) -> str:
+        return VERTEX_INTERACTIONS_API_VERSION
+
+    def get_default_vertex_location(self) -> str:
+        return VERTEX_INTERACTIONS_DEFAULT_LOCATION
+
+    def _mint(self, litellm_params: GenericLiteLLMParams) -> tuple[str, str]:
+        raw_params: Final = litellm_params.model_dump()
+        return self._mint_access_token(
+            self.safe_get_vertex_ai_credentials(raw_params),
+            self.safe_get_vertex_ai_project(raw_params),
+        )
+
+    def _target(self, api_base: str | None, litellm_params: GenericLiteLLMParams) -> VertexInteractionsTarget:
+        _, project_id = self._mint(litellm_params)
+        if not project_id:
+            raise ValueError(
+                "Vertex AI project is required. Set vertex_project, litellm.vertex_project, or VERTEXAI_PROJECT"
+            )
+        location: Final = validate_vertex_location(
+            self.explicit_vertex_ai_location(litellm_params.model_dump()) or VERTEX_INTERACTIONS_DEFAULT_LOCATION
+        )
+        return VertexInteractionsTarget(
+            base_url=self.get_api_base(api_base or None, location),
+            project_id=project_id,
+            location=location,
+        )
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        litellm_params: GenericLiteLLMParams | None,
+    ) -> dict:  # mutable-ok: BaseInteractionsAPIConfig declares plain-dict headers
+        access_token, _ = self._mint(litellm_params or GenericLiteLLMParams())
+        return {  # mutable-ok: BaseInteractionsAPIConfig declares plain-dict headers
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {access_token}",
+            **headers,
+        }
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        model: str | None,
+        agent: str | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+        stream: bool | None = None,
+    ) -> str:
+        params: Final = (
+            GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams()
+        )
+        collection_url: Final = self._target(api_base, params).collection_url
+        return f"{collection_url}?alt=sse" if stream else collection_url
+
+    def _interaction_by_id_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        url_suffix: str = "",
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        target: Final = self._target(api_base or None, litellm_params)
+        return f"{target.interaction_url(interaction_id)}{url_suffix}", {}  # mutable-ok: same base contract
+
+    def transform_get_interaction_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        return self._interaction_by_id_request(interaction_id, api_base, litellm_params)
+
+    def transform_delete_interaction_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        return self._interaction_by_id_request(interaction_id, api_base, litellm_params)
+
+    def transform_cancel_interaction_request(
+        self,
+        interaction_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict]:  # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body
+        return self._interaction_by_id_request(interaction_id, api_base, litellm_params, url_suffix=":cancel")

litellm/llms/vertex_ai/videos/transformation.py

@@ -7,11 +7,11 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer
 
 import base64
 import time
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
 
 import httpx
-from httpx._types import RequestFiles
+from httpx._types import FileContent, RequestFiles
 from typing_extensions import ReadOnly
 
 from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
@@ -677,9 +677,10 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, object] | None = None,
         prefetched_source_data: dict[str, Any] | None = None,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         """
         Build a predictLongRunning edit request from the pre-fetched source video.
 
@@ -727,7 +728,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
                 request_data["parameters"] = vertex_params
 
         edit_url: Final = f"{api_base.rstrip('/')}/{model}:predictLongRunning"
-        return edit_url, request_data
+        return edit_url, request_data, None
 
     def transform_video_edit_response(
         self,

litellm/main.py

@@ -24,6 +24,7 @@ from concurrent import futures
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from copy import deepcopy
 from functools import partial
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
 
 from litellm._logging import _redact_string
@@ -416,7 +417,7 @@ async def acompletion(
     logprobs: bool | None = None,
     top_logprobs: int | None = None,
     deployment_id=None,
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
@@ -602,7 +603,7 @@ async def acompletion(
         _, custom_llm_provider, _, _ = get_llm_provider(
             model=model,
             custom_llm_provider=custom_llm_provider,
-            api_base=base_url,
+            api_base=kwargs.get("api_base") or base_url,
         )
 
     fallbacks = fallbacks or litellm.model_fallbacks
@@ -1811,6 +1812,56 @@ def _complete_fireworks_ai(
     return response
 
 
+def _complete_together_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+    acompletion: Final = ctx.acompletion
+    api_base: Final = ctx.api_base
+    api_key: Final = ctx.api_key
+    client: Final = _dispatch_client_http(ctx)
+    custom_llm_provider: Final = ctx.custom_llm_provider
+    headers: Final = ctx.headers
+    litellm_params: Final = ctx.litellm_params
+    logging: Final = ctx.logging
+    messages: Final = ctx.messages
+    model: Final = ctx.model
+    model_response: Final = ctx.model_response
+    optional_params: Final = ctx.optional_params
+    provider_config: Final = ctx.provider_config
+    shared_session: Final = ctx.shared_session
+    stream: Final = ctx.stream
+    timeout: Final = ctx.timeout
+
+    try:
+        response: Final = base_llm_http_handler.completion(
+            model=model,
+            messages=messages,
+            headers=headers,
+            model_response=model_response,
+            api_key=api_key,
+            api_base=api_base,
+            acompletion=acompletion,
+            logging_obj=logging,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            shared_session=shared_session,
+            timeout=timeout,
+            client=client,
+            custom_llm_provider=custom_llm_provider,
+            encoding=_get_encoding(),
+            stream=stream,
+            provider_config=provider_config,
+        )
+    except Exception as e:
+        logging.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=str(e),
+            additional_args=MappingProxyType({"headers": headers}),
+        )
+        raise
+
+    return response
+
+
 def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
     acompletion: Final = ctx.acompletion
     api_base: Final = ctx.api_base
@@ -4920,7 +4971,7 @@ def completion(
     logit_bias: dict | None = None,
     user: str | None = None,
     # openai v1.0+ new params
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     response_format: dict | type[BaseModel] | None = None,
     seed: int | None = None,
@@ -5600,6 +5651,8 @@ def completion(
         elif custom_llm_provider == "fireworks_ai":
             ## COMPLETION CALL
             response = _complete_fireworks_ai(_dispatch_ctx)
+        elif custom_llm_provider == "together_ai":
+            response = _complete_together_ai(_dispatch_ctx)
         elif custom_llm_provider == "heroku":
             response = _complete_heroku(_dispatch_ctx)
 
@@ -5649,7 +5702,6 @@ def completion(
             or custom_llm_provider == "volcengine"
             or custom_llm_provider == "anyscale"
             or custom_llm_provider == "openai"
-            or custom_llm_provider == "together_ai"
             or custom_llm_provider == "nebius"
             or custom_llm_provider == "wandb"
             or custom_llm_provider == "clarifai"
@@ -5699,14 +5751,6 @@ def completion(
             response = _complete_openrouter(_dispatch_ctx)
         elif custom_llm_provider == "vercel_ai_gateway":
             response = _complete_vercel_ai_gateway(_dispatch_ctx)
-        elif (
-            custom_llm_provider == "together_ai"
-            or ("togethercomputer" in model)
-            or (model in litellm.together_ai_models)
-        ):
-            """
-            Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility
-            """
         elif custom_llm_provider == "palm":
             raise ValueError(
                 "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en"

litellm/model_prices_and_context_window_backup.json

@@ -17155,6 +17155,14 @@
             "notes": "Web Search on Amazon Bedrock AgentCore, billed by AWS on the gateway"
         }
     },
+    "bing_grounding/search": {
+        "input_cost_per_query": 0.035,
+        "litellm_provider": "bing_grounding",
+        "mode": "search",
+        "metadata": {
+            "notes": "Grounding with Bing Search (G1 SKU): $35 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment."
+        }
+    },
     "tinyfish/search": {
         "input_cost_per_query": 0.0,
         "litellm_provider": "tinyfish",
@@ -37878,6 +37886,7 @@
         "output_cost_per_token": 1e-07
     },
     "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-06",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -37894,6 +37903,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": {
+        "deprecation_date": "2026-07-10",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262000,
@@ -37906,6 +37916,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
+        "deprecation_date": "2026-04-16",
         "input_cost_per_token": 6.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
@@ -37918,6 +37929,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 40000,
@@ -37929,6 +37941,7 @@
         "supports_tool_choice": false
     },
     "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+        "deprecation_date": "2026-06-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
@@ -37941,11 +37954,15 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-R1": {
+        "deprecation_date": "2026-05-14",
         "input_cost_per_token": 3e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
         "max_output_tokens": 20480,
         "max_tokens": 20480,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 7e-06,
         "supports_function_calling": true,
@@ -37954,6 +37971,7 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": {
+        "deprecation_date": "2026-02-03",
         "input_cost_per_token": 5.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
@@ -37971,6 +37989,9 @@
         "max_input_tokens": 65536,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.25e-06,
         "supports_function_calling": true,
@@ -37979,9 +38000,13 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-V3.1": {
+        "deprecation_date": "2026-05-14",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "max_tokens": 16384,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.7e-06,
         "source": "https://www.together.ai/models/deepseek-v3-1",
@@ -37993,6 +38018,7 @@
         "max_output_tokens": 16384
     },
     "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": {
+        "deprecation_date": "2026-03-06",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38001,16 +38027,21 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": {
-        "input_cost_per_token": 8.8e-07,
+        "input_cost_per_token": 1.04e-06,
         "litellm_provider": "together_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
         "mode": "chat",
-        "output_cost_per_token": 8.8e-07,
+        "output_cost_per_token": 1.04e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": {
+        "deprecation_date": "2025-11-13",
         "input_cost_per_token": 0,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38021,6 +38052,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
+        "deprecation_date": "2026-03-31",
         "input_cost_per_token": 2.7e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38031,6 +38063,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38041,6 +38074,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 3.5e-06,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38051,6 +38085,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-25",
         "input_cost_per_token": 8.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38061,6 +38096,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": {
+        "deprecation_date": "2026-03-06",
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38071,6 +38107,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mistral-7B-Instruct-v0.1": {
+        "deprecation_date": "2025-11-13",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38079,6 +38116,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
+        "deprecation_date": "2026-04-02",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38086,6 +38124,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": {
+        "deprecation_date": "2026-04-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38098,6 +38137,9 @@
     "together_ai/moonshotai/Kimi-K2-Instruct": {
         "input_cost_per_token": 1e-06,
         "litellm_provider": "together_ai",
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "source": "https://www.together.ai/models/kimi-k2-instruct",
@@ -38141,6 +38183,7 @@
         "supports_tool_choice": true
     },
     "together_ai/zai-org/GLM-4.5-Air-FP8": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
@@ -38158,6 +38201,9 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 200000,
         "max_tokens": 200000,
+        "metadata": {
+            "successor": "together_ai/zai-org/GLM-5.2"
+        },
         "mode": "chat",
         "output_cost_per_token": 2.2e-06,
         "source": "https://www.together.ai/models/glm-4-6",
@@ -38167,11 +38213,15 @@
         "supports_tool_choice": true
     },
     "together_ai/zai-org/GLM-4.7": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 200000,
         "max_output_tokens": 200000,
         "max_tokens": 200000,
+        "metadata": {
+            "successor": "together_ai/zai-org/GLM-5.2"
+        },
         "mode": "chat",
         "output_cost_per_token": 2e-06,
         "source": "https://www.together.ai/models/glm-4-7",
@@ -38181,11 +38231,15 @@
         "supports_tool_choice": true
     },
     "together_ai/moonshotai/Kimi-K2.5": {
+        "deprecation_date": "2026-05-21",
         "input_cost_per_token": 5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
         "max_output_tokens": 256000,
         "max_tokens": 256000,
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 2.8e-06,
         "source": "https://www.together.ai/models/kimi-k2-5",
@@ -38195,9 +38249,13 @@
         "supports_reasoning": true
     },
     "together_ai/moonshotai/Kimi-K2-Instruct-0905": {
+        "deprecation_date": "2026-03-06",
         "input_cost_per_token": 1e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "source": "https://www.together.ai/models/kimi-k2-0905",
@@ -38206,9 +38264,13 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/Qwen/Qwen3.7-Plus"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.5e-06,
         "source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct",
@@ -38218,9 +38280,13 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": {
+        "deprecation_date": "2026-02-25",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/Qwen/Qwen3.6-Plus"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.5e-06,
         "source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking",
@@ -38230,6 +38296,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3.5-397B-A17B": {
+        "deprecation_date": "2026-06-29",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
@@ -38241,6 +38308,292 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "together_ai/MiniMaxAI/MiniMax-M3": {
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/Prism-ML/Ternary-Bonsai-27B": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.5-9B": {
+        "input_cost_per_token": 1.7e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/Qwen/Qwen3.6-Plus": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_reasoning": true
+    },
+    "together_ai/Qwen/Qwen3.7-Max": {
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3.75e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.7-Plus": {
+        "input_cost_per_token": 3.2e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 1.28e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1010000,
+        "max_output_tokens": 1010000,
+        "max_tokens": 1010000,
+        "mode": "chat",
+        "output_cost_per_token": 6.25e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/arize-ai/qwen-2-1.5b-instruct": {
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro": {
+        "input_cost_per_token": 1.74e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 512000,
+        "max_output_tokens": 512000,
+        "max_tokens": 512000,
+        "mode": "chat",
+        "output_cost_per_token": 3.48e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
+        "input_cost_per_token": 1.32e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 3.96e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/google/gemma-3n-E4B-it": {
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/google/gemma-4-31B-it": {
+        "input_cost_per_token": 3.9e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 9.7e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/intfloat/multilingual-e5-large-instruct": {
+        "input_cost_per_token": 2e-08,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 514,
+        "max_tokens": 514,
+        "mode": "embedding",
+        "output_cost_per_token": 2e-08,
+        "output_vector_size": 1024,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/meta-llama/Llama-Guard-4-12B": {
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 2e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/meta-models/Muse-Glimmer-30B": {
+        "input_cost_per_token": 3.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/moonshotai/Kimi-K2.7-Code": {
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/moonshotai/Kimi-K3": {
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 512288,
+        "max_output_tokens": 512288,
+        "max_tokens": 512288,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/pearl-ai/gemma-4-31b-it": {
+        "input_cost_per_token": 2.8e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.6e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/thinkingmachines/Inkling": {
+        "input_cost_per_token": 1e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 4.05e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/thinkingmachines/Inkling-Small": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/zai-org/GLM-5.2": {
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048575,
+        "max_output_tokens": 1048575,
+        "max_tokens": 1048575,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "tts-1": {
         "input_cost_per_character": 1.5e-05,
         "litellm_provider": "openai",
@@ -49008,12 +49361,13 @@
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
         "litellm_provider": "bedrock_mantle",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
         "use_openai_responses_path": true,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ],
         "supported_modalities": [
@@ -49040,12 +49394,13 @@
         "output_cost_per_token": 1.32e-05,
         "output_cost_per_token_above_272k_tokens": 1.98e-05,
         "litellm_provider": "bedrock_mantle",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
         "use_openai_responses_path": true,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ],
         "supported_modalities": [
@@ -49072,12 +49427,13 @@
         "output_cost_per_token": 1.32e-06,
         "output_cost_per_token_above_272k_tokens": 1.98e-06,
         "litellm_provider": "bedrock_mantle",
-        "max_input_tokens": 1000000,
+        "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
         "use_openai_responses_path": true,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ],
         "supported_modalities": [

litellm/models/team.py

@@ -64,8 +64,8 @@ class TeamBase(LiteLLMPydanticObjectBase):
     team_alias: str | None = None
     team_id: str | None = None
     organization_id: str | None = None
-    admins: list = []
-    members: list = []
+    admins: list[str] = []
+    members: list[str] = []
     members_with_roles: list[Member] = []
     team_member_permissions: list[str] | None = None
     metadata: dict | None = None
@@ -75,7 +75,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
     soft_budget: float | None = None
     budget_duration: str | None = None
     budget_limits: list[BudgetLimitEntry] | None = None
-    models: list = []
+    models: list[str] = []
     blocked: bool = False
     router_settings: dict | None = None
     access_group_ids: list[str] | None = None

litellm/passthrough/main.py

@@ -199,7 +199,7 @@ def llm_passthrough_route(
         api_key=api_key,
     )
 
-    litellm_params_dict: Final = get_litellm_params(**kwargs)
+    litellm_params_dict: Final = get_litellm_params(api_key=api_key, api_base=api_base, **kwargs)
 
     if client is None:
         from litellm.llms.custom_httpx.http_handler import (

litellm/proxy/_experimental/mcp_server/db.py

@@ -4,7 +4,7 @@ import hashlib
 import json
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -13,7 +13,6 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request
 from litellm.proxy._types import (
     LiteLLM_MCPServerTable,
-    LiteLLM_ObjectPermissionTable,
     MCPApprovalStatus,
     MCPEnvVar,
     MCPEnvVarScope,
@@ -30,6 +29,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
 )
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     MCPServerOAuthClientRepository,
     MCPServerRepository,
@@ -48,34 +48,9 @@ if TYPE_CHECKING:
 
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
-_RowT = TypeVar("_RowT")
-
-
-class _TableActions(Protocol[_RowT]):
-    async def find_unique(
-        self, where: Mapping[str, object], include: Mapping[str, object] | None = None
-    ) -> _RowT | None: ...
-
-    async def find_many(
-        self,
-        take: int | None = None,
-        where: Mapping[str, object] | None = None,
-        order: Mapping[str, object] | None = None,
-    ) -> list[_RowT]: ...
-
-    async def create(self, data: Mapping[str, object]) -> _RowT: ...
-
-    async def upsert(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT: ...
-
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT | None: ...
-
-    async def delete(self, where: Mapping[str, object]) -> _RowT | None: ...
-
-    async def delete_many(self, where: Mapping[str, object] | None = None) -> int: ...
-
 
 class _UserEnvVarsTransactionClient(Protocol):
-    litellm_mcpuserenvvars: "_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]"
+    litellm_mcpuserenvvars: "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]"
 
     async def execute_raw(self, query: str, *args: object) -> int: ...
 
@@ -473,15 +448,15 @@ def _credentials_blob_to_mutable_dict(blob: str | Mapping[str, object]) -> dict[
 
 def _mcp_server_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPServerTable]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPServerTable]] = MCPServerRepository(prisma_client).table
+) -> "TableActions[prisma_db_models.LiteLLM_MCPServerTable]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPServerTable]] = MCPServerRepository(prisma_client).table
     return table
 
 
 def _verification_token_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_VerificationToken]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_VerificationToken]] = VerificationTokenRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_VerificationToken]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_VerificationToken]] = VerificationTokenRepository(
         prisma_client
     ).table
     return table
@@ -489,15 +464,15 @@ def _verification_token_table_actions(
 
 def _team_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_TeamTable]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
+) -> "TableActions[prisma_db_models.LiteLLM_TeamTable]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
     return table
 
 
 def _oauth_client_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = MCPServerOAuthClientRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = MCPServerOAuthClientRepository(
         prisma_client
     ).table
     return table
@@ -511,7 +486,7 @@ def _db_transaction_manager(prisma_client: PrismaClient) -> _UserEnvVarsTransact
 async def _db_find_mcp_server_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPServerTableWhereInput | None" = None,
-) -> "list[prisma_db_models.LiteLLM_MCPServerTable]":
+) -> "Sequence[prisma_db_models.LiteLLM_MCPServerTable]":
     return await _mcp_server_table_actions(prisma_client).find_many(where=where)
 
 
@@ -526,17 +501,19 @@ async def _db_update_mcp_server_row(
     server_id: str,
     data: "prisma_db_types.LiteLLM_MCPServerTableUpdateInput",
 ) -> "prisma_db_models.LiteLLM_MCPServerTable":
-    row: Final[prisma_db_models.LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.update(
+    row: Final[prisma_db_models.LiteLLM_MCPServerTable | None] = await _mcp_server_table_actions(prisma_client).update(
         where={"server_id": server_id},
         data=data,
     )
+    if row is None:
+        raise ValueError(f"MCP server not found, passed server_id={server_id}")
     return row
 
 
 def _user_credential_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]] = MCPUserCredentialsRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]] = MCPUserCredentialsRepository(
         prisma_client
     ).table
     return table
@@ -544,8 +521,8 @@ def _user_credential_actions(
 
 def _user_env_var_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars
+) -> "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars
     return table
 
 
@@ -560,7 +537,7 @@ async def _db_find_user_credential_row(
 async def _db_find_user_credential_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPUserCredentialsWhereInput | None" = None,
-) -> "list[prisma_db_models.LiteLLM_MCPUserCredentials]":
+) -> "Sequence[prisma_db_models.LiteLLM_MCPUserCredentials]":
     return await _user_credential_actions(prisma_client).find_many(where=where)
 
 
@@ -583,7 +560,7 @@ async def _db_upsert_user_credential_row(
 async def _db_find_user_env_var_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPUserEnvVarsWhereInput | None" = None,
-) -> "list[prisma_db_models.LiteLLM_MCPUserEnvVars]":
+) -> "Sequence[prisma_db_models.LiteLLM_MCPUserEnvVars]":
     return await _user_env_var_actions(prisma_client).find_many(where=where)
 
 
@@ -658,7 +635,7 @@ async def get_mcp_servers(prisma_client: PrismaClient, server_ids: Iterable[str]
     """
     Returns the matching mcp servers from the db with the server_ids
     """
-    _mcp_servers: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
+    _mcp_servers: Final[Sequence[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
         prisma_client
     ).find_many(
         where={
@@ -745,13 +722,13 @@ async def get_all_mcp_servers_for_user(
 
 async def get_objectpermissions_for_mcp_server(
     prisma_client: PrismaClient, mcp_server_id: str
-) -> list[LiteLLM_ObjectPermissionTable]:
+) -> "Sequence[prisma_db_models.LiteLLM_ObjectPermissionTable]":
     """
     Get all the object permissions records and the associated team and verficiationtoken records that have access to the mcp server
     """
-    object_permission_records: Final[list[LiteLLM_ObjectPermissionTable]] = await ObjectPermissionRepository(
-        prisma_client
-    ).table.find_many(
+    object_permission_records: Final[
+        Sequence[prisma_db_models.LiteLLM_ObjectPermissionTable]
+    ] = await ObjectPermissionRepository(prisma_client).table.find_many(
         where={
             "mcp_servers": {"has": mcp_server_id},
         },
@@ -766,19 +743,19 @@ async def get_objectpermissions_for_mcp_server(
 
 async def get_virtualkeys_for_mcp_server(
     prisma_client: PrismaClient, server_id: str
-) -> "list[prisma_db_models.LiteLLM_VerificationToken]":
+) -> "Sequence[prisma_db_models.LiteLLM_VerificationToken]":
     """
     Get all the virtual keys that have access to the mcp server
     """
-    virtual_keys: Final[list[prisma_db_models.LiteLLM_VerificationToken] | None] = await VerificationTokenRepository(
-        prisma_client
-    ).table.find_many(
+    virtual_keys: Final[
+        Sequence[prisma_db_models.LiteLLM_VerificationToken] | None
+    ] = await VerificationTokenRepository(prisma_client).table.find_many(
         where={
             "mcp_servers": {"has": server_id},
         },
     )
 
-    if virtual_keys is None:
+    if virtual_keys is None:  # pyright: ignore[reportUnnecessaryComparison]  # unreachable per seam types; kept as-is
         return []
     return virtual_keys
 
@@ -860,7 +837,7 @@ async def delete_mcp_server(
                 invalidate_token_cache = global_mcp_server_manager.invalidate_user_oauth_token_cache
             for user_id in credential_user_ids:
                 await invalidate_token_cache(user_id, server_id)
-    return deleted_server
+    return deleted_server  # pyright: ignore[reportReturnType]  # prisma row, not domain LiteLLM_MCPServerTable
 
 
 async def create_mcp_server(
@@ -880,7 +857,7 @@ async def create_mcp_server(
     data_dict["updated_by"] = touched_by
 
     new_mcp_server: Final[LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.create(
-        data=data_dict
+        data=data_dict,  # pyright: ignore[reportAssignmentType]  # prisma row, not domain LiteLLM_MCPServerTable
     )
 
     _decrypt_env_vars_on_returned_row(new_mcp_server)
@@ -982,7 +959,7 @@ async def update_mcp_server(
     data: UpdateMCPServerRequest,
     touched_by: str,
     fields_set: set[str] | None = None,
-) -> LiteLLM_MCPServerTable:
+) -> LiteLLM_MCPServerTable | None:
     """
     Update a new mcp server record in the db
     """
@@ -1093,9 +1070,9 @@ async def update_mcp_server(
 
         data_dict["credentials"] = Json(None)
 
-    updated_mcp_server: Final[LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.update(
+    updated_mcp_server: Final[LiteLLM_MCPServerTable | None] = await MCPServerRepository(prisma_client).table.update(
         where={"server_id": data.server_id},
-        data=data_dict,
+        data=data_dict,  # pyright: ignore[reportAssignmentType]  # prisma row, not domain LiteLLM_MCPServerTable
     )
 
     _decrypt_env_vars_on_returned_row(updated_mcp_server)
@@ -1181,7 +1158,7 @@ async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient,
         )
         updated += 1
 
-    oauth_clients: Final[list[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = await _oauth_client_table_actions(
+    oauth_clients: Final[Sequence[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = await _oauth_client_table_actions(
         prisma_client
     ).find_many()
     oauth_updated = 0
@@ -1914,7 +1891,7 @@ async def get_mcp_submissions(
     along with a summary count breakdown by approval_status.
     Mirrors get_guardrail_submissions() from guardrail_endpoints.py.
     """
-    rows: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
+    rows: Final[Sequence[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
         prisma_client
     ).find_many(
         where={"submitted_at": {"not": None}},

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
+2:I[347257,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
+3:I[871135,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js","/my-custom-path/_next/static/chunks/37t2cfzl_b58p.js","/my-custom-path/_next/static/chunks/1zf358k334atp.js","/my-custom-path/_next/static/chunks/3kil-7y33kpm9.js","/my-custom-path/_next/static/chunks/2i0218zvrsasm.js","/my-custom-path/_next/static/chunks/3ytz29phknzsy.js","/my-custom-path/_next/static/chunks/0aoel7yrv88fp.js","/my-custom-path/_next/static/chunks/1natmx9lu3mus.js","/my-custom-path/_next/static/chunks/1dmg55q8kht9j.js","/my-custom-path/_next/static/chunks/1lrl_8p0h2sbm.js","/my-custom-path/_next/static/chunks/1e-4-g6x6zyse.js","/my-custom-path/_next/static/chunks/0gygfcpmiijl8.js","/my-custom-path/_next/static/chunks/2cu4j3g1tldv4.js","/my-custom-path/_next/static/chunks/2kmqjpt047tjo.js","/my-custom-path/_next/static/chunks/1phty1k2nx8fx.js","/my-custom-path/_next/static/chunks/0u3cfuz-tf0wj.js","/my-custom-path/_next/static/chunks/39s4-rh6l9sa1.js","/my-custom-path/_next/static/chunks/0dylouuq8ak8p.js","/my-custom-path/_next/static/chunks/3a3jpg95umjho.js","/my-custom-path/_next/static/chunks/28hnu_qv5e_c_.js","/my-custom-path/_next/static/chunks/0qf1_0kt4uuxa.js","/my-custom-path/_next/static/chunks/0ab_ntohf1wik.js","/my-custom-path/_next/static/chunks/3kpec-qy1uzod.js"],"default"]
+6:I[897367,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/my-custom-path/_next/static/chunks/1e-4-g6x6zyse.js","async":true}],["$","script","script-1",{"src":"/my-custom-path/_next/static/chunks/0gygfcpmiijl8.js","async":true}],["$","script","script-2",{"src":"/my-custom-path/_next/static/chunks/2cu4j3g1tldv4.js","async":true}],["$","script","script-3",{"src":"/my-custom-path/_next/static/chunks/2kmqjpt047tjo.js","async":true}],["$","script","script-4",{"src":"/my-custom-path/_next/static/chunks/1phty1k2nx8fx.js","async":true}],["$","script","script-5",{"src":"/my-custom-path/_next/static/chunks/0u3cfuz-tf0wj.js","async":true}],["$","script","script-6",{"src":"/my-custom-path/_next/static/chunks/39s4-rh6l9sa1.js","async":true}],["$","script","script-7",{"src":"/my-custom-path/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/my-custom-path/_next/static/chunks/3a3jpg95umjho.js","async":true}],["$","script","script-9",{"src":"/my-custom-path/_next/static/chunks/28hnu_qv5e_c_.js","async":true}],["$","script","script-10",{"src":"/my-custom-path/_next/static/chunks/0qf1_0kt4uuxa.js","async":true}],["$","script","script-11",{"src":"/my-custom-path/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/my-custom-path/_next/static/chunks/3kpec-qy1uzod.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +1,7 @@
 1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+2:I[92825,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
+3:I[216370,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js","/my-custom-path/_next/static/chunks/37t2cfzl_b58p.js","/my-custom-path/_next/static/chunks/1zf358k334atp.js","/my-custom-path/_next/static/chunks/3kil-7y33kpm9.js","/my-custom-path/_next/static/chunks/2i0218zvrsasm.js","/my-custom-path/_next/static/chunks/3ytz29phknzsy.js","/my-custom-path/_next/static/chunks/0aoel7yrv88fp.js","/my-custom-path/_next/static/chunks/1natmx9lu3mus.js","/my-custom-path/_next/static/chunks/1dmg55q8kht9j.js","/my-custom-path/_next/static/chunks/1lrl_8p0h2sbm.js"],"default"]
+4:I[339756,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[837457,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/my-custom-path/_next/static/chunks/37t2cfzl_b58p.js","async":true}],["$","script","script-1",{"src":"/my-custom-path/_next/static/chunks/1zf358k334atp.js","async":true}],["$","script","script-2",{"src":"/my-custom-path/_next/static/chunks/3kil-7y33kpm9.js","async":true}],["$","script","script-3",{"src":"/my-custom-path/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/my-custom-path/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/my-custom-path/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-6",{"src":"/my-custom-path/_next/static/chunks/1natmx9lu3mus.js","async":true}],["$","script","script-7",{"src":"/my-custom-path/_next/static/chunks/1dmg55q8kht9j.js","async":true}],["$","script","script-8",{"src":"/my-custom-path/_next/static/chunks/1lrl_8p0h2sbm.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
 6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +1,6 @@
 1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
+2:I[897367,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
+3:I[897367,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
+5:I[27201,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
 0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,11 +1,11 @@
 1:"$Sreact.fragment"
-2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
-3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
-4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
-6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
-:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
+2:I[363178,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
+3:I[12985,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
+4:I[867271,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[557951,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
+6:I[339756,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+7:I[837457,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+8:I[713354,["/my-custom-path/_next/static/chunks/3_06chgeyldml.js","/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
+:HL["/my-custom-path/_next/static/chunks/1kid9zr1--h6y.css","style"]
+:HL["/my-custom-path/_next/static/chunks/2hicgq-mjp8vy.css","style"]
+0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/2hicgq-mjp8vy.css","precedence":"next"}],["$","script","script-0",{"src":"/my-custom-path/_next/static/chunks/3_06chgeyldml.js","async":true}],["$","script","script-1",{"src":"/my-custom-path/_next/static/chunks/26h-ny89yaww0.js","async":true}],["$","script","script-2",{"src":"/my-custom-path/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
-:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
-:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
+:HL["/my-custom-path/_next/static/chunks/1kid9zr1--h6y.css","style"]
+:HL["/my-custom-path/_next/static/chunks/2hicgq-mjp8vy.css","style"]
+:HL["/my-custom-path/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
 0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_buildManifest.js

@@ -3,7 +3,7 @@ self.__BUILD_MANIFEST = {
     "afterFiles": [],
     "beforeFiles": [
       {
-        "source": "/litellm-asset-prefix/_next/:path+",
+        "source": "/my-custom-path/_next/:path+",
         "destination": "/_next/:path+"
       }
     ],