Merge remote-tracking branch 'origin/main' into litellm_unified_key_policy_hook

.circleci/config.yml

@@ -147,6 +147,9 @@ commands:
       db_name:
         type: string
         default: circle_test
+      image:
+        type: string
+        default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
     steps:
       - run:
           name: Start PostgreSQL
@@ -157,7 +160,7 @@ commands:
               -e POSTGRES_PASSWORD=postgres \
               -e POSTGRES_DB=<< parameters.db_name >> \
               -p 5432:5432 \
-              postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
+              << parameters.image >>
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
@@ -1084,9 +1087,7 @@ jobs:
           name: Run tests
           command: |
             mkdir -p test-results
-            TEST_FILES=$(printf "%s\n%s\n" \
-              "$(circleci tests glob "tests/ocr_tests/**/test_*.py")" \
-              "tests/test_litellm/ocr/test_rust_bridge.py")
+            TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
@@ -2914,7 +2915,69 @@ jobs:
               exit 1
             fi
 
+  provider_replay_harness:
+    docker:
+      - *python312_image
+    working_directory: ~/project
+    resource_class: medium
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Test provider replay harness
+          command: |
+            mkdir -p test-results/provider-replay-harness
+            uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
+              --junitxml=test-results/provider-replay-harness/junit.xml \
+              tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
+              tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
+              tests/code_coverage_tests/test_provider_replay_harness.py
+      - store_test_results:
+          path: test-results/provider-replay-harness
+
+  integration_contracts:
+    parameters:
+      suite:
+        type: string
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+      - start_redis
+      - run:
+          name: Run owned integration contracts
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          no_output_timeout: 15m
+      - run:
+          name: Stop owned database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/integration-<< parameters.suite >>
+            docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
 workflows:
+  integration:
+    jobs:
+      - integration_contracts:
+          name: integration-<< matrix.suite >>
+          matrix:
+            parameters:
+              suite: [management, accounting, providers]
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
   build_and_test:
     jobs:
       - using_litellm_on_windows:
@@ -2923,6 +2986,7 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - provider_replay_harness
       - base_sdk_install:
           filters: *main_branches
       - local_testing_part1:

.circleci/scripts/run_integration.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+suite="${1:?integration suite required}"
+results="test-results/integration-${suite}"
+mkdir -p "$results"
+integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')"
+upstream_pid=""
+proxy_pid=""
+peer_pid=""
+launched_pid=""
+guard_created=false
+guard_installed=false
+guard6_created=false
+guard6_installed=false
+cleanup() {
+  original_status=$?
+  trap - EXIT INT TERM
+  sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \
+    "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \
+    > "$results/process-cleanup.txt" 2>&1 || original_status=1
+  for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do
+    if [ -n "$owned_pid" ]; then
+      kill -- "-$owned_pid" 2>/dev/null || true
+      for _ in {1..50}; do
+        kill -0 -- "-$owned_pid" 2>/dev/null || break
+        sleep 0.1
+      done
+      if kill -0 -- "-$owned_pid" 2>/dev/null; then
+        kill -KILL -- "-$owned_pid" 2>/dev/null || true
+        original_status=1
+      fi
+      wait "$owned_pid" 2>/dev/null || true
+    fi
+  done
+  if [ "$guard_installed" = true ]; then
+    sudo iptables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+  fi
+  if [ "$guard_created" = true ]; then
+    sudo iptables -F integration_only || original_status=1
+    sudo iptables -X integration_only || original_status=1
+  fi
+  if [ "$guard6_installed" = true ]; then
+    sudo ip6tables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+  fi
+  if [ "$guard6_created" = true ]; then
+    sudo ip6tables -F integration_only || original_status=1
+    sudo ip6tables -X integration_only || original_status=1
+  fi
+  printf '%s\n' "$original_status" > "$results/exit-status.txt"
+  exit "$original_status"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+export PATH="$PWD/.venv/bin:$PATH"
+export PYTHONPATH="$PWD:$PWD/tests:$PWD/tests/e2e"
+export DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:5432/circle_test"
+export REDIS_HOST=127.0.0.1 REDIS_PORT=6379
+export LITELLM_MASTER_KEY=sk-integration-master LITELLM_SALT_KEY=sk-integration-salt
+export LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True
+export STORE_MODEL_IN_DB=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1
+export INTEGRATION_PROXY_URL=http://127.0.0.1:4000
+export INTEGRATION_PEER_URL=""
+export INTEGRATION_UPSTREAM_URL=http://127.0.0.1:8190
+export INTEGRATION_MASTER_KEY="$LITELLM_MASTER_KEY"
+export INTEGRATION_SEED="$((16#$(git rev-parse --short=8 HEAD)))"
+
+uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
+
+sudo iptables -N integration_only
+guard_created=true
+sudo iptables -A integration_only -o lo -j ACCEPT
+sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+for service in postgres-db redis-cache; do
+  address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
+  sudo iptables -A integration_only -d "$address" -j ACCEPT
+done
+sudo iptables -A integration_only -j REJECT
+sudo iptables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+guard_installed=true
+sudo ip6tables -N integration_only
+guard6_created=true
+sudo ip6tables -A integration_only -o lo -j ACCEPT
+sudo ip6tables -A integration_only -j REJECT
+sudo ip6tables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+guard6_installed=true
+
+if curl --noproxy '*' --connect-timeout 2 -s http://198.51.100.1 >/dev/null 2>&1; then
+  echo "Unexpected outbound network access" >&2
+  exit 1
+fi
+sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
+awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
+
+setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
+  .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
+upstream_pid=$!
+start_proxy() {
+  local port="$1"
+  local log_name="$2"
+  setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
+    DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+    LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" \
+    LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True STORE_MODEL_IN_DB=True \
+    AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
+    .venv/bin/python -m integration._support.proxy --config tests/integration/proxy_config.yaml \
+    --host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \
+    --use_prisma_db_push --enforce_prisma_migration_check \
+    > "$results/$log_name" 2>&1 &
+  launched_pid=$!
+}
+start_proxy 4000 proxy.log
+proxy_pid="$launched_pid"
+.venv/bin/python .circleci/scripts/wait_integration_services.py
+if [ "$suite" = management ]; then
+  export INTEGRATION_PEER_URL=http://127.0.0.1:4001
+  start_proxy 4001 peer.log
+  peer_pid="$launched_pid"
+  .venv/bin/python .circleci/scripts/wait_integration_services.py
+fi
+
+if [ "$suite" = providers ]; then
+  INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --noconftest -o addopts= \
+    --strict-markers --strict-config -p no:pytest-retry -p no:rerunfailures --timeout=30 \
+    tests/e2e/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
+    tests/e2e/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
+    tests/e2e/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
+    tests/e2e/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
+    --junitxml="$results/replay-controls.xml"
+fi
+
+timeout --signal=TERM --kill-after=20s 11m env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
+  INTEGRATION_RUN_ID="$integration_identity" \
+  DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+  INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
+  INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
+  INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
+  INTEGRATION_SEED="$INTEGRATION_SEED" \
+  LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results"

.circleci/scripts/stop_integration_processes.py

@@ -0,0 +1,53 @@
+import sys
+from typing import Final
+
+import psutil
+
+
+def is_owned(process: psutil.Process, identity: str, owner_uid: int) -> bool:
+    try:
+        return process.uids().real == owner_uid and process.environ().get("INTEGRATION_RUN_ID") == identity
+    except psutil.NoSuchProcess:
+        return False
+
+
+def owned_processes(identity: str, owner_uid: int) -> tuple[psutil.Process, ...]:
+    return tuple(process for process in psutil.process_iter() if is_owned(process, identity, owner_uid))
+
+
+def main(identity: str, owner_uid: int, root_pids: tuple[int, ...]) -> int:
+    assert owner_uid > 0, "The integration process owner must be a non-root UID"
+    owned: Final = owned_processes(identity, owner_uid)
+    roots: Final = tuple(process for process in owned if process.pid in root_pids)
+    for process in roots:
+        try:
+            process.terminate()
+        except psutil.NoSuchProcess:
+            continue
+    psutil.wait_procs(roots, timeout=30)
+    residual: Final = owned_processes(identity, owner_uid)
+    for process in residual:
+        try:
+            process.terminate()
+        except psutil.NoSuchProcess:
+            continue
+    psutil.wait_procs(residual, timeout=10)
+    remaining: Final = owned_processes(identity, owner_uid)
+    for process in remaining:
+        try:
+            process.kill()
+        except psutil.NoSuchProcess:
+            continue
+    psutil.wait_procs(remaining, timeout=2)
+    survivors: Final = owned_processes(identity, owner_uid)
+    print(
+        f"Owned integration processes: {len(owned)}, roots: {len(roots)}, "
+        f"residual: {len(residual)}, forced: {len(remaining)}, remaining: {len(survivors)}"
+    )
+    for process in remaining:
+        print(f"Forced cleanup was required for PID {process.pid}")
+    return 1 if remaining or survivors else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1], int(sys.argv[2]), tuple(int(value) for value in sys.argv[3:] if value)))

.circleci/scripts/wait_integration_services.py

@@ -0,0 +1,43 @@
+import os
+import time
+from typing import Final
+
+import httpx
+from redis import Redis
+
+
+def main() -> None:
+    primary: Final = os.environ["INTEGRATION_PROXY_URL"]
+    peer: Final = os.environ.get("INTEGRATION_PEER_URL")
+    proxies: Final = (primary, peer) if peer else (primary,)
+    deadline: Final = time.monotonic() + 90
+    headers: Final = {"Authorization": f"Bearer {os.environ['INTEGRATION_MASTER_KEY']}"}
+    with httpx.Client(trust_env=False, timeout=2) as client, Redis(
+        host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2
+    ) as cache:
+        while True:
+            try:
+                ready: Final = (
+                    client.get(f"{os.environ['INTEGRATION_UPSTREAM_URL']}/health").status_code == 200
+                    and all(client.get(f"{url}/health/readiness").status_code == 200 for url in proxies)
+                )
+                if ready:
+                    for url in proxies:
+                        response: Final = client.get(f"{url}/cache/ping", headers=headers)
+                        response.raise_for_status()
+                        result: Final = response.json()
+                        assert result["status"] == "healthy", result
+                        assert result["cache_type"] == "redis", result
+                        assert result["ping_response"] is True, result
+                        assert result["set_cache_response"] == "success", result
+                    if cache.pubsub_numsub("litellm_proxy.auth_cache_invalidation")[0][1] >= len(proxies):
+                        return
+            except httpx.TransportError:
+                pass
+            if time.monotonic() >= deadline:
+                raise SystemExit("Integration services or auth-cache subscribers did not become ready")
+            time.sleep(0.2)
+
+
+if __name__ == "__main__":
+    main()

.github/codeql/codeql-config.yml

@@ -14,6 +14,17 @@ query-filters:
       id: py/clear-text-logging-sensitive-data  # CWE-312
   - exclude:
       id: py/polynomial-redos                   # CWE-730
+  # Import resolution confuses stdlib types with management_endpoints/types.py.
+  # The generic cycle query also reports intentional deferred imports.
+  - exclude:
+      id: py/cyclic-import
+  - exclude:
+      id: py/unsafe-cyclic-import
+  # Known false positives on live settings and Protocol placeholders.
+  - exclude:
+      id: py/unused-global-variable
+  - exclude:
+      id: py/ineffectual-statement
 
 paths-ignore:
   - tests

.github/e2e-stack/assert_tests_ran.py

@@ -3,6 +3,9 @@ import xml.etree.ElementTree as ET
 from pathlib import Path
 from typing import Final
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
+from coverage_registry.management_cases import MANAGEMENT_CASES
+
 
 def main() -> int:
     selected: Final = tuple(sys.argv[2:])
@@ -16,6 +19,17 @@ def main() -> int:
         case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
     )
     missing: Final = tuple(path for path in selected if path not in passed)
+    required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
+    passed_nodes: Final = frozenset(
+        prop.get("value")
+        for case in cases
+        if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+        for prop in case.findall("./properties/property")
+        if prop.get("name") == "management_node"
+    )
+    missing_nodes: Final = required_nodes - passed_nodes
+    for node in sorted(missing_nodes):
+        _ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
     for path in selected:
         collected: Final = sum(case.get("file") == path for case in cases)
         skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
@@ -27,6 +41,7 @@ def main() -> int:
     if (
         selected
         and not missing
+        and not missing_nodes
         and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
     ):
         return 0

.github/e2e-stack/oidc-profile.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "${REPO_ROOT}"
+exec uv run --no-sync python tests/e2e/idp.py "$@"

.github/e2e-stack/select_tests.py

@@ -12,6 +12,8 @@ UNSUPPORTED: Final = re.compile(
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
     r"|^tests/e2e/idp_realm\.json$"
+    r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
+    r"|^tests/e2e/coverage_registry/management_cases\.py$"
     r"|^tests/e2e/gateway/"
     r"|^\.github/e2e-stack/"
     r"|^\.github/workflows/test-e2e-changed\.yml$"

.github/pull_request_template.md

@@ -47,6 +47,10 @@ After: the same request comes back with real token counts, so the dashboard show
 
 <!-- e.g., "Fixes #000" -->
 
+## Affected release
+
+<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Leave the section blank otherwise -->
+
 ## Linear ticket
 
 <!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
@@ -154,3 +158,4 @@ Example checklists:
 ## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
+

.github/scripts/assert_ci_coverage.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import ast
+import json
 import operator
 import pathlib
 import re
@@ -498,6 +499,69 @@ def _check_shards() -> int:
     return 0
 
 
+def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
+    manifest: Final = repo_root / "tests/integration/contracts.json"
+    if not manifest.exists():
+        return frozenset(), ()
+    entries: Final = json.loads(manifest.read_text())
+    paths: Final = frozenset(node.split("::", 1)[0] for node in entries["tests"])
+    circle_path: Final = repo_root / ".circleci/config.yml"
+    circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
+    steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
+    invoked: Final = any(
+        ".circleci/scripts/run_integration.sh" in scalar.value
+        for scalar in _scalars(steps, "integration_contracts")
+        if scalar.key == "command"
+    )
+    scheduled: Final = frozenset(
+        suite
+        for job in circle.get("workflows", {}).get("integration", {}).get("jobs", ())
+        if isinstance(job, dict) and "integration_contracts" in job
+        for suite in job["integration_contracts"]
+        .get("matrix", {})
+        .get("parameters", {})
+        .get("suite", (job["integration_contracts"].get("suite"),))
+        if isinstance(suite, str)
+    )
+    required: Final = frozenset(
+        group
+        for group, folders in entries["groups"].items()
+        if any(any(path.startswith(f"tests/integration/{folder}/") for folder in folders) for path in paths)
+    )
+    ungrouped: Final = frozenset(
+        path
+        for path in paths
+        if sum(
+            any(path.startswith(f"tests/integration/{folder}/") for folder in folders)
+            for folders in entries["groups"].values()
+        )
+        != 1
+    )
+    gha_tokens: Final = _invoked_test_tokens(
+        scalar
+        for path in (repo_root / ".github/workflows").glob("*.y*ml")
+        for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
+    )
+    findings: Final = tuple(
+        Finding(path, "integration contract is also selected by GitHub Actions")
+        for path in paths
+        if any(_token_covers(token, path) for token in gha_tokens)
+    ) + tuple(
+        Finding(path, "canonical integration test file is missing")
+        for path in paths
+        if not (repo_root / path).is_file()
+    )
+    group_findings: Final = tuple(
+        Finding(group, "canonical integration group is not scheduled by CircleCI")
+        for group in sorted(required - scheduled)
+    ) + tuple(Finding(path, "canonical node must have exactly one integration group") for path in sorted(ungrouped))
+    if not paths or not invoked or not scheduled:
+        return frozenset(), findings + (
+            Finding(str(manifest.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
+        )
+    return paths, findings + group_findings
+
+
 def main() -> int:
     if "--shards" in sys.argv[1:]:
         return _check_shards()
@@ -507,7 +571,8 @@ def main() -> int:
     allowlist = _load_allowlist()
     scalars = _all_scalars()
 
-    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
+    integration_paths, ownership_findings = _integration_ownership()
+    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | integration_paths) + ownership_findings
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
     stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
 

.github/scripts/auto_update_price_and_context_window_file.py

@@ -1,6 +1,8 @@
 import asyncio
 import aiohttp
 import json
+import math
+from typing import Any
 
 # Asynchronously fetch data from a given URL
 async def fetch_data(url):
@@ -21,11 +23,157 @@ async def fetch_data(url):
         print("Error fetching data from URL:", e)
         return None
 
+
+FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models"
+FRIENDLI_PROVIDER = "friendliai"
+
+INHERITABLE_BASE_KEYS = (
+    "supports_pdf_input",
+    "supports_assistant_prefill",
+    "supports_adaptive_thinking",
+    "supports_output_config",
+)
+
+REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
+
+
+def _find_base_model_entry(base_model: str, local_data: dict) -> str | None:
+    if not base_model:
+        return None
+    bm_tail = base_model.split("/")[-1].lower()
+    if base_model in local_data:
+        return base_model
+    for key in local_data:
+        if key.startswith("sample_spec") or key == "fallback_generalizations":
+            continue
+        if key.split("/")[-1].lower() == bm_tail:
+            return key
+    return None
+
+
+def _reasoning_effort_levels(reasoning_options: list) -> list:
+    offered = {
+        val
+        for opt in reasoning_options or []
+        if opt.get("type") == "effort"
+        for val in opt.get("values", [])
+    }
+    return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered]
+
+
+def _valid_token_price(value: object) -> bool:
+    try:
+        price = float(value)  # pyright: ignore[reportArgumentType]  # non-numeric values are rejected via the except
+    except (TypeError, ValueError):
+        return False
+    return math.isfinite(price) and price >= 0
+
+
+def _has_valid_token_prices(pricing: dict | None) -> bool:
+    prices = pricing or {}
+    return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output"))
+
+
+def _pricing(pricing: dict) -> dict:
+    out: dict[str, Any] = {}
+    if not pricing:
+        return out
+    if "input" in pricing:
+        out["input_cost_per_token"] = float(pricing["input"])
+    if "output" in pricing:
+        out["output_cost_per_token"] = float(pricing["output"])
+    if "input_cache_read" in pricing and pricing["input_cache_read"] is not None:
+        out["cache_read_input_token_cost"] = float(pricing["input_cache_read"])
+    return out
+
+
+def _modality_flags(input_mods: list) -> dict:
+    mods = input_mods or []
+    has_image = "image" in mods
+    return {
+        "supports_vision": has_image,
+        "supports_image_input": has_image,
+        "supports_video_input": "video" in mods,
+    }
+
+
+def transform_friendli_data(data: list, local_data: dict) -> dict:
+    transformed: dict[str, dict] = {}
+    if not data:
+        return transformed
+    for model in data:
+        # An unpriced row must never wholesale-replace an already priced local entry:
+        # missing prices cost-calculate as zero, silently zeroing tracked spend
+        if not _has_valid_token_prices(model.get("pricing")):
+            continue
+        model_id = model["id"]
+        base_model = model.get("base_model") or ""
+        entry: dict[str, Any] = {
+            "litellm_provider": FRIENDLI_PROVIDER,
+        }
+
+        base_key = _find_base_model_entry(base_model, local_data)
+        if base_key:
+            base_entry = local_data[base_key]
+            for k in INHERITABLE_BASE_KEYS:
+                if k in base_entry:
+                    entry[k] = base_entry[k]
+
+        ctx = model.get("context_length")
+        if ctx is not None:
+            entry["max_input_tokens"] = int(ctx)
+        max_out = model.get("max_completion_tokens")
+        if max_out is not None:
+            entry["max_output_tokens"] = int(max_out)
+            entry["max_tokens"] = int(max_out)
+
+        pricing = _pricing(model.get("pricing", {}))
+        entry.update(pricing)
+        entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing
+
+        reasoning = model.get("reasoning") is True
+        entry["supports_reasoning"] = reasoning
+        if reasoning:
+            entry["reasoning_effort_levels"] = _reasoning_effort_levels(
+                model.get("reasoning_options", [])
+            )
+
+        func = model.get("functionality", {})
+        entry["supports_function_calling"] = func.get("tool_call") is True
+        entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True
+        is_struct = func.get("structured_output") is True
+        entry["supports_response_schema"] = is_struct
+        entry["supports_native_structured_output"] = is_struct
+        entry["supports_system_messages"] = func.get("system_messages") is True
+        entry["supports_tool_choice"] = func.get("tool_choice") is True
+
+        entry.update(_modality_flags(model.get("input_modalities", [])))
+
+        entry["mode"] = model.get("mode", "chat")
+
+        desc = model.get("description")
+        if desc:
+            entry["comment"] = desc
+
+        dep = model.get("deprecation_date")
+        if dep:
+            entry["deprecation_date"] = dep.split("T")[0]
+
+        entry["source"] = FRIENDLI_API_URL
+
+        transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry
+    return transformed
+
 # Synchronize local data with remote data
-def sync_local_data_with_remote(local_data, remote_data):
+def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
     # Update existing keys in local_data with values from remote_data
+    # (replace_keys entries are swapped wholesale so a field the remote catalog
+    #  dropped, e.g. cache pricing, cannot survive as a stale value)
     for key in (set(local_data) & set(remote_data)):
-        local_data[key].update(remote_data[key])
+        if key in replace_keys:
+            local_data[key] = remote_data[key]
+        else:
+            local_data[key].update(remote_data[key])
 
     # Add new keys from remote_data to local_data
     for key in (set(remote_data) - set(local_data)):
@@ -46,6 +194,8 @@ def write_to_file(file_path, data):
 # Update the existing models and add the missing models for OpenRouter
 def transform_openrouter_data(data):
     transformed = {}
+    if not data:
+        return transformed
     for row in data:
         # Add the fields 'max_tokens' and 'input_cost_per_token'
         obj = {
@@ -84,7 +234,14 @@ def transform_openrouter_data(data):
 # Update the existing models and add the missing models for Vercel AI Gateway
 def transform_vercel_ai_gateway_data(data):
     transformed = {}
+    if not data:
+        return transformed
     for row in data:
+        # Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync
+        if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any(
+            row.get("pricing", {}).get(k) is None for k in ("input", "output")
+        ):
+            continue
         obj = {
             "max_tokens": row["context_window"],
             "input_cost_per_token": float(row["pricing"]["input"]),
@@ -143,13 +300,16 @@ def main():
     vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
     # Transform the fetched Vercel AI Gateway data
     vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+
+    friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
+    friendli_data = transform_friendli_data(friendli_data, local_data)
     
     # Combine both datasets
-    all_remote_data = {**openrouter_data, **vercel_data}
+    all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
 
     # If both local and openrouter data are available, synchronize and save
     if local_data and all_remote_data:
-        sync_local_data_with_remote(local_data, all_remote_data)
+        sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")

.github/workflows/_test-unit-base.yml

@@ -57,6 +57,7 @@ permissions:
 
 env:
   UV_PYTHON: "3.12"
+  LITELLM_LOCAL_MODEL_COST_MAP: "True"
 
 jobs:
   run:
@@ -113,6 +114,7 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         run: |
+          diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
 

.github/workflows/guard-main-branch.yml

@@ -1,42 +0,0 @@
-name: Guard main branch
-
-on:
-  pull_request:
-    branches:
-      - main
-  merge_group:
-
-permissions: {}
-
-# DO NOT RENAME the job's `name:` — it is referenced by GitHub branch
-# protection as a required status check on `main`. Renaming silently
-# breaks the gate.
-jobs:
-  guard:
-    name: Verify PR source branch
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - name: Reject merge_group events
-        if: github.event_name == 'merge_group'
-        run: |
-          echo "::error::Merge queue is not supported for main. Disable merge queue or update this guard."
-          exit 1
-      - name: Check head branch name
-        env:
-          HEAD_REF: ${{ github.head_ref }}
-          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
-          BASE_REPO: ${{ github.repository }}
-        run: |
-          echo "PR head repo: $HEAD_REPO"
-          echo "PR head branch: $HEAD_REF"
-          if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
-            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against 'litellm_internal_staging' instead."
-            exit 1
-          fi
-          if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
-            echo "Allowed source branch."
-            exit 0
-          fi
-          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_internal_staging' instead."
-          exit 1

.github/workflows/test-code-quality.yml

@@ -178,7 +178,7 @@ jobs:
           version: "0.10.9"
 
       - name: Install dependencies
-        run: uv sync --frozen --extra proxy --python 3.10
+        run: uv sync --frozen --extra proxy --extra cli --python 3.10
 
       - run: uv run --no-sync python --version
 
@@ -187,3 +187,6 @@ jobs:
 
       - name: Check litellm CLI
         run: uv run --no-sync litellm --version
+
+      - name: Check lite CLI
+        run: uv run --no-sync lite version

.github/workflows/test-e2e-changed.yml

@@ -183,7 +183,7 @@ jobs:
             log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
               -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
             status=$?
             uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -27,10 +27,13 @@ import litellm
 from litellm import Router, verbose_logger
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
+from litellm.constants import MAX_FILE_LIST_LIMIT
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
+from openai.types.file_deleted import FileDeleted
+
 from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
@@ -48,7 +51,6 @@ from litellm.proxy._types import (
 from litellm.proxy.openai_files_endpoints.common_utils import (
     BATCH_CREATE_HIDDEN_PARAM,
     FILE_LIST_CONTINUATION_CHUNK_SIZE,
-    MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
     decode_model_from_file_id,
@@ -1787,7 +1789,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         litellm_parent_otel_span: Optional[Span],
         llm_router: Router,
         **data: Dict,
-    ) -> OpenAIFileObject:
+    ) -> FileDeleted:
 
         # Check if file deletion should be blocked due to batch references
         await self._check_file_deletion_allowed(file_id)
@@ -1795,7 +1797,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         # file_id = convert_b64_uid_to_unified_uid(file_id)
         model_file_id_mapping = await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span)
 
-        delete_response = None
         specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
         if specific_model_file_id_mapping:
             # Remove conflicting keys from data to avoid duplicate keyword arguments
@@ -1810,23 +1811,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         else {}
                     ),
                 }
-                delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
+                await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
 
-        stored_file_object = await self.delete_unified_file_id(file_id, litellm_parent_otel_span)
+        await self.delete_unified_file_id(file_id, litellm_parent_otel_span)
 
-        # Record successful deletion metric only on actual success
-        if stored_file_object or delete_response:
-            prom_logger = self._get_prometheus_logger()
-            if prom_logger:
-                prom_logger.record_managed_file_deleted(result="success")
-
-        if stored_file_object:
-            return OpenAIFileObject.model_validate(stored_file_object).model_copy(update={"id": file_id})
-        elif delete_response:
-            delete_response.id = file_id
-            return delete_response
-        else:
-            raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
+        prom_logger = self._get_prometheus_logger()
+        if prom_logger:
+            prom_logger.record_managed_file_deleted(result="success")
+        return FileDeleted(id=file_id, object="file", deleted=True)
 
     async def afile_content(
         self,

litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120000_add_litellm_call_id_spend_logs/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "litellm_call_id" TEXT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120001_spend_logs_litellm_call_id_index/migration.sql

@@ -0,0 +1,12 @@
+-- CreateIndex (CONCURRENTLY)
+--
+-- Disclaimer:
+-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
+--   single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
+-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
+--   interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
+-- - Do not edit this file after it has been applied to any database: Prisma checksums
+--   migrations; add a new migration instead.
+-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
+--   without IF NOT EXISTS if you must support older versions).
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -37,11 +37,13 @@ raised it above the deploy default keeps that larger budget for deploy unless
 the deploy override says otherwise.
 """
 
+import importlib.util
 import math
 import os
 import shutil
 import signal
 import subprocess
+import sys
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
@@ -64,6 +66,7 @@ DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
 DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0
 
 BOOTSTRAP_ARG = "--version"
+PRISMA_CONSOLE_SCRIPT = "prisma"
 
 
 @dataclass(frozen=True)
@@ -184,6 +187,28 @@ def _kill_process_group(process: "subprocess.Popen[str]") -> None:
         return
 
 
+def prisma_cli_available() -> bool:
+    """Whether some way of running the Prisma CLI exists: the console script on PATH or the importable package."""
+    if shutil.which(PRISMA_CONSOLE_SCRIPT) is not None:
+        return True
+    return importlib.util.find_spec(PRISMA_CONSOLE_SCRIPT) is not None
+
+
+def resolve_prisma_argv(argv: Sequence[str]) -> tuple[str, ...]:
+    """Route a bare ``prisma`` command through ``python -m prisma`` when the console script is not on PATH.
+
+    The console script and ``python -m prisma`` are the same entry point, but
+    only the module form survives an interpreter whose ``bin`` directory is
+    missing from PATH, which is how the proxy gets started under launchers and
+    init systems. Any other executable name is left untouched.
+    """
+    if not argv or argv[0] != PRISMA_CONSOLE_SCRIPT:
+        return tuple(argv)
+    if shutil.which(PRISMA_CONSOLE_SCRIPT) is not None:
+        return tuple(argv)
+    return (sys.executable, "-m", PRISMA_CONSOLE_SCRIPT, *argv[1:])
+
+
 def run_prisma(
     argv: Sequence[str],
     *,
@@ -200,7 +225,7 @@ def run_prisma(
     text unless ``stdout``/``stderr`` say otherwise.
     """
     with subprocess.Popen(
-        argv,
+        resolve_prisma_argv(argv),
         env=env,
         stdout=stdout,
         stderr=stderr,

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -659,12 +659,14 @@ model LiteLLM_SpendLogs {
   mcp_namespaced_tool_name String? 
   agent_id            String?
   proxy_server_request Json?     @default("{}")
+  litellm_call_id     String?
   created_at          DateTime   @default(now()) @map("created_at")
   updated_at          DateTime   @default(now()) @updatedAt @map("updated_at")
   @@index([startTime])
   @@index([startTime, request_id])
   @@index([end_user])
   @@index([session_id])
+  @@index([litellm_call_id])
 }
 
 model LiteLLM_BudgetWindowSpend {

litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs

@@ -5,6 +5,7 @@ use serde_json::Value;
 
 #[derive(Deserialize)]
 struct Input {
+    path: String,
     model_alias: String,
     provider_model: String,
     api_base: String,
@@ -21,7 +22,8 @@ async fn main() {
         Ok(input) => input,
         Err(error) => fail(error),
     };
-    let result = litellm_ai_gateway::trace_parity::traced_messages_request(
+    let result = litellm_ai_gateway::trace_parity::traced_request(
+        input.path,
         input.model_alias,
         input.provider_model,
         input.api_base,

litellm-rust/crates/ai-gateway/src/trace_parity.rs

@@ -29,14 +29,15 @@ pub struct TracedGatewayResponse {
     pub trace: Vec<litellm_core::observability::FunctionTraceEvent>,
 }
 
-pub async fn traced_messages_request(
+pub async fn traced_request(
+    path: String,
     model_alias: String,
     provider_model: String,
     api_base: String,
     body: Value,
 ) -> TracedGatewayResponse {
     let trace = litellm_core::observability::FunctionTrace::default();
-    let result = messages_request(model_alias, provider_model, api_base, body)
+    let result = request(path, model_alias, provider_model, api_base, body)
         .with_subscriber(trace.dispatcher())
         .await;
     let events = trace.events();
@@ -54,7 +55,8 @@ pub async fn traced_messages_request(
     }
 }
 
-pub async fn messages_request(
+pub async fn request(
+    path: String,
     model_alias: String,
     provider_model: String,
     api_base: String,
@@ -75,7 +77,7 @@ pub async fn messages_request(
     };
     let request = Request::builder()
         .method("POST")
-        .uri("/v1/messages")
+        .uri(path)
         .header(AUTHORIZATION, "Bearer trace-master-key")
         .header(CONTENT_TYPE, "application/json")
         .body(Body::from(body.to_string()))

litellm-rust/crates/python-bridge/src/routes/ocr/callbacks.rs

@@ -58,6 +58,13 @@ impl PythonLogger {
                 params.set_item(name, value)?;
             }
         }
+        for name in custom_pricing_fields(py)? {
+            if let Some(value) = kwargs.bind(py).get_item(&name)?
+                && !value.is_none()
+            {
+                params.set_item(name, value)?;
+            }
+        }
         update.set_item("litellm_params", params)?;
         update.set_item("custom_llm_provider", &pre_call.custom_llm_provider)?;
         self.object(py)
@@ -120,6 +127,17 @@ impl PythonLogger {
     }
 }
 
+fn custom_pricing_fields(py: Python<'_>) -> PyResult<Vec<String>> {
+    py.import("litellm.types.utils")?
+        .getattr("CustomPricingLiteLLMParams")?
+        .getattr("model_fields")?
+        .cast_into::<PyDict>()?
+        .keys()
+        .iter()
+        .map(|name| name.extract::<String>())
+        .collect()
+}
+
 fn redact(
     py: Python<'_>,
     params: &Bound<'_, PyDict>,

litellm/__init__.py

@@ -501,6 +501,7 @@ disable_copilot_system_to_assistant: bool = False  # If false (default), convert
 public_mcp_servers: Optional[List[str]] = None
 public_mcp_hub_strict_whitelist: bool = True
 public_model_groups: Optional[List[str]] = None
+public_skills_index: bool = False
 public_agent_groups: Optional[List[str]] = None
 agent_search_embedding_model: Optional[str] = None
 mcp_tool_search: Optional[Mapping[str, object]] = None

litellm/_logging.py

@@ -1,5 +1,6 @@
 import ast
 import contextvars
+import functools
 import logging
 import os
 import sys
@@ -225,6 +226,35 @@ class AccessLogRedactionFilter(logging.Filter):
 _access_log_filter: Final = AccessLogRedactionFilter()
 
 
+@functools.lru_cache(maxsize=1)
+def _parse_disabled_access_log_paths(raw: str) -> frozenset[str]:
+    return frozenset(stripped for path in raw.split(",") if (stripped := path.strip()))
+
+
+def _disabled_access_log_paths() -> frozenset[str]:
+    """Read the variable per record so a value loaded later via proxy config
+    environment_variables or dotenv is honored."""
+    return _parse_disabled_access_log_paths(os.getenv("LITELLM_DISABLE_ACCESS_LOG_PATHS", ""))
+
+
+class AccessLogPathFilter(logging.Filter):
+    """Drops uvicorn.access records for request paths listed in LITELLM_DISABLE_ACCESS_LOG_PATHS.
+
+    uvicorn passes record.args as (client_addr, method, full_path, http_version, status_code).
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not isinstance(record.args, tuple) or len(record.args) < 3:
+            return True
+        full_path: Final = record.args[2]
+        if not isinstance(full_path, str):
+            return True
+        return full_path.partition("?")[0] not in _disabled_access_log_paths()
+
+
+_access_log_path_filter: Final = AccessLogPathFilter()
+
+
 def _get_max_string_length_stdout_log() -> int:
     """Read the limit per record so a value loaded later via proxy config
     environment_variables is honored."""
@@ -663,6 +693,7 @@ def _redact_third_party_loggers() -> None:
     for name in _REDACTED_THIRD_PARTY_LOGGERS:
         logging.getLogger(name).addFilter(_secret_filter)
     for name in _REDACTED_ACCESS_LOGGERS:
+        logging.getLogger(name).addFilter(_access_log_path_filter)
         logging.getLogger(name).addFilter(_access_log_filter)
 
 

litellm/batches/batch_utils.py

@@ -9,6 +9,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
+from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import ModelInfo, Usage
 from litellm.utils import token_counter
@@ -356,6 +357,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
             prompt_tokens=_prompt,
             completion_tokens=_completion,
             total_tokens=_total,
+            prompt_tokens_details=vertex_prompt_tokens_details(usage_metadata),
         )
 
         try:

litellm/caching/caching.py

@@ -679,7 +679,14 @@ class Cache:
             cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs)
             self.cache.set_cache(cache_key, cached_data, **kwargs)
         except Exception as e:
-            log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Cache: exception in add_cache", e)
+            self._log_add_cache_failure(e)
+
+    def _log_add_cache_failure(self, exc: Exception) -> None:
+        message: Final = "LiteLLM Cache: exception in add_cache"
+        if isinstance(self.cache, RedisCache):
+            log_redis_failure(verbose_logger, logging.ERROR, message, exc)
+            return
+        verbose_logger.error("%s: %s", message, exc)
 
     async def async_add_cache(self, result, dynamic_cache_object: BaseCache | None = None, **kwargs):
         """
@@ -698,7 +705,7 @@ class Cache:
                 else:
                     await self.cache.async_set_cache(cache_key, cached_data, **kwargs)
         except Exception as e:
-            log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Cache: exception in add_cache", e)
+            self._log_add_cache_failure(e)
 
     def _convert_to_cached_embedding(
         self,
@@ -877,7 +884,7 @@ class Cache:
             else:
                 await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs)
         except Exception as e:
-            log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Cache: exception in add_cache", e)
+            self._log_add_cache_failure(e)
 
     def should_use_cache(self, **kwargs):
         """

litellm/caching/redis_cache.py

@@ -15,8 +15,9 @@ import hashlib
 import inspect
 import json
 import logging
+import threading
 import time
-from collections.abc import Awaitable, Callable, Sequence
+from collections.abc import Awaitable, Callable, Iterator, Sequence
 from contextvars import ContextVar
 from dataclasses import dataclass
 from datetime import timedelta
@@ -32,6 +33,7 @@ from litellm.constants import (
     REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
     REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
     REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
+    REDIS_TIMEOUT_LOG_INTERVAL,
 )
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
@@ -317,19 +319,37 @@ def _is_redis_health_failure(exc: BaseException) -> bool:
 def _redis_timeout_error_types() -> tuple[type, ...]:
     """Health failures that are timeouts rather than unambiguous connectivity errors.
 
-    ``builtins.TimeoutError`` covers ``asyncio.TimeoutError`` and ``socket.timeout``
-    (aliases since py3.11 / py3.10). ``redis.exceptions.TimeoutError`` does not subclass
-    either, so it is listed explicitly.
+    ``builtins.TimeoutError`` covers ``socket.timeout`` (an alias since py3.10) and, from
+    py3.11, ``asyncio.TimeoutError``; on py3.10 ``asyncio.TimeoutError`` is still its own
+    class, so it is listed explicitly. ``redis.exceptions.TimeoutError`` subclasses neither.
     """
     try:
         from redis.exceptions import TimeoutError as RedisTimeoutError
     except ImportError:
-        return (TimeoutError,)
-    return (RedisTimeoutError, TimeoutError)
+        return (TimeoutError, asyncio.TimeoutError)
+    return (RedisTimeoutError, TimeoutError, asyncio.TimeoutError)
 
 
-def _is_redis_timeout_failure(exc: BaseException) -> bool:
-    return isinstance(exc, _redis_timeout_error_types())
+_MAX_EXCEPTION_CAUSE_DEPTH: Final = 20
+
+
+def _explicit_causes(exc: BaseException) -> Iterator[BaseException]:
+    current = exc  # rebind-ok: advances one link per iteration of the bounded walk
+    for _ in range(_MAX_EXCEPTION_CAUSE_DEPTH):
+        yield current
+        if current.__cause__ is None:
+            return
+        current = current.__cause__
+
+
+def is_redis_timeout_failure(exc: BaseException) -> bool:
+    """True when ``exc`` or any exception it was explicitly raised ``from`` is a timeout.
+
+    redis-py's blocking pool reports a pool wait timeout as ``ConnectionError`` chained from
+    ``asyncio.TimeoutError``, which is a busy pool rather than an unreachable Redis.
+    """
+    timeout_types: Final = _redis_timeout_error_types()
+    return any(isinstance(link, timeout_types) for link in _explicit_causes(exc))
 
 
 class _BreakerMetrics:
@@ -396,7 +416,7 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep
     """
     if not _is_redis_health_failure(exc):
         return
-    breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc))
+    breaker.record_failure(is_timeout=is_redis_timeout_failure(exc))
     _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1)
 
 
@@ -404,13 +424,58 @@ class RedisCircuitBreakerOpenError(Exception):
     pass
 
 
+class _RedisTimeoutLogThrottle:
+    """Admits one Redis timeout log line per interval and counts the timeouts it suppressed in between."""
+
+    def __init__(self, interval: float, clock: Callable[[], float] = time.monotonic) -> None:
+        self.interval = interval
+        self._clock = clock
+        self._lock = threading.Lock()
+        self._last_logged_at: float | None = None
+        self._suppressed = 0
+
+    def admit(self) -> int | None:
+        """Return the number of timeouts suppressed since the last admitted line, or None to suppress this one."""
+        with self._lock:
+            now: Final = self._clock()
+            if self._last_logged_at is not None and now - self._last_logged_at < self.interval:
+                self._suppressed += 1
+                return None
+            suppressed: Final = self._suppressed
+            self._suppressed = 0
+            self._last_logged_at = now
+            return suppressed
+
+
+_redis_timeout_log_throttle: Final = _RedisTimeoutLogThrottle(REDIS_TIMEOUT_LOG_INTERVAL)
+
+
 def log_redis_failure(
     logger: logging.Logger, level: int, message: str, exc: BaseException, with_traceback: bool = False
 ) -> None:
     if isinstance(exc, RedisCircuitBreakerOpenError):
-        logger.debug("%s: %s", message, exc)
+        logger.debug("%s: %s", message, exc, stacklevel=2)
         return
-    logger.log(level, "%s: %s", message, exc, exc_info=exc if with_traceback else None)
+    exc_info: Final = exc if with_traceback else None
+    if not is_redis_timeout_failure(exc):
+        logger.log(level, "%s: %s", message, exc, exc_info=exc_info, stacklevel=2)
+        return
+    suppressed: Final = _redis_timeout_log_throttle.admit()
+    if suppressed is None:
+        logger.debug("%s: %s", message, exc, stacklevel=2)
+        return
+    if suppressed == 0:
+        logger.log(level, "%s: %s", message, exc, exc_info=exc_info, stacklevel=2)
+        return
+    logger.log(
+        level,
+        "%s: %s (%d more Redis timeouts since the previous Redis timeout line were logged at DEBUG)",
+        message,
+        exc,
+        suppressed,
+        exc_info=exc_info,
+        stacklevel=2,
+    )
 
 
 @dataclass(frozen=True, slots=True)
@@ -457,7 +522,7 @@ async def _run_under_circuit_breaker(
         result: Final = await call()
     except Exception as e:
         if _is_redis_health_failure(e):
-            breaker.record_failure(is_timeout=_is_redis_timeout_failure(e))
+            breaker.record_failure(is_timeout=is_redis_timeout_failure(e))
         raise
     _exit_circuit_breaker(breaker, admission)
     return result
@@ -474,7 +539,7 @@ def _run_under_circuit_breaker_sync(
         result: Final = call()
     except Exception as e:
         if _is_redis_health_failure(e):
-            breaker.record_failure(is_timeout=_is_redis_timeout_failure(e))
+            breaker.record_failure(is_timeout=is_redis_timeout_failure(e))
         raise
     _exit_circuit_breaker(breaker, admission)
     return result
@@ -783,10 +848,8 @@ class RedisCache(BaseCache):
             ## LOGGING ##
             end_time = time.time()
             _duration = end_time - start_time
-            verbose_logger.error(
-                "LiteLLM Redis Caching: increment_cache() - Got exception from REDIS %s, Writing value=%s",
-                str(e),
-                value,
+            log_redis_failure(
+                verbose_logger, logging.ERROR, "LiteLLM Redis Caching: increment_cache() - Got exception from REDIS", e
             )
             raise e
 
@@ -992,11 +1055,8 @@ class RedisCache(BaseCache):
                     call_type=f"async_set_cache <- {_get_call_stack_info()}",
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r",
-                str(e),
-                key,
-                value,
+            log_redis_failure(
+                verbose_logger, logging.ERROR, "LiteLLM Redis Caching: async set() - Got exception from REDIS", e
             )
             raise e
 
@@ -1044,10 +1104,8 @@ class RedisCache(BaseCache):
                     event_metadata={"key": key},
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
-                str(e),
-                value,
+            log_redis_failure(
+                verbose_logger, logging.ERROR, "LiteLLM Redis Caching: async set() - Got exception from REDIS", e
             )
             _record_swallowed_redis_failure(self._circuit_breaker, e)
 
@@ -1094,7 +1152,6 @@ class RedisCache(BaseCache):
         start_time: Final = time.time()
 
         print_verbose(f"Set Async Redis Cache: key list: {cache_list}\nttl={ttl}, redis_version={self.redis_version}")
-        cache_value: Final = None
         try:
             async with _redis_client.pipeline(transaction=False) as pipe:
                 results: Final = await self._pipeline_helper(pipe, cache_list, ttl)
@@ -1131,10 +1188,11 @@ class RedisCache(BaseCache):
                 )
             )
 
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS %s, Writing value=%s",
-                str(e),
-                cache_value,
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS",
+                e,
             )
             _record_swallowed_redis_failure(self._circuit_breaker, e)
 
@@ -1173,8 +1231,11 @@ class RedisCache(BaseCache):
                     end_time=time.time(),
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async_set_cache_pipeline_with_ttls() - Got exception from REDIS %s", str(e)
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async_set_cache_pipeline_with_ttls() - Got exception from REDIS",
+                e,
             )
             _record_swallowed_redis_failure(self._circuit_breaker, e)
 
@@ -1217,10 +1278,8 @@ class RedisCache(BaseCache):
                 )
             )
             # NON blocking - notify users Redis is throwing an exception
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
-                str(e),
-                value,
+            log_redis_failure(
+                verbose_logger, logging.ERROR, "LiteLLM Redis Caching: async set() - Got exception from REDIS", e
             )
             raise e
 
@@ -1256,10 +1315,11 @@ class RedisCache(BaseCache):
                 )
             )
             # NON blocking - notify users Redis is throwing an exception
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async set_cache_sadd() - Got exception from REDIS %s, Writing value=%s",
-                str(e),
-                value,
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async set_cache_sadd() - Got exception from REDIS",
+                e,
             )
             _record_swallowed_redis_failure(self._circuit_breaker, e)
 
@@ -1341,10 +1401,11 @@ class RedisCache(BaseCache):
                     parent_otel_span=parent_otel_span,
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async async_increment() - Got exception from REDIS %s, Writing value=%s",
-                str(e),
-                value,
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async async_increment() - Got exception from REDIS",
+                e,
             )
             raise e
 
@@ -1430,7 +1491,9 @@ class RedisCache(BaseCache):
             print_verbose(f"Got Redis Cache: key: {key}, cached_response {cached_response}")
             return self._get_cache_logic(cached_response=cached_response)
         except Exception as e:
-            verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e)
+            log_redis_failure(
+                verbose_logger, logging.ERROR, "litellm.caching.caching: get() - Got exception from REDIS", e
+            )
             _record_swallowed_redis_failure(self._circuit_breaker, e)
 
     def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
@@ -1508,7 +1571,7 @@ class RedisCache(BaseCache):
                 end_time=failed_at,
                 parent_otel_span=parent_otel_span,
             )
-            verbose_logger.error("Error occurred in batch get cache - %s", e)
+            log_redis_failure(verbose_logger, logging.ERROR, "Error occurred in batch get cache", e)
             _record_swallowed_redis_failure(self._circuit_breaker, e)
             return key_value_dict
 
@@ -1627,7 +1690,7 @@ class RedisCache(BaseCache):
                     parent_otel_span=parent_otel_span,
                 )
             )
-            verbose_logger.error("Error occurred in async batch get cache - %s", e)
+            log_redis_failure(verbose_logger, logging.ERROR, "Error occurred in async batch get cache", e)
             _record_swallowed_redis_failure(self._circuit_breaker, e)
             return key_value_dict
 
@@ -1852,9 +1915,11 @@ class RedisCache(BaseCache):
                     parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async increment_pipeline() - Got exception from REDIS %s",
-                str(e),
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async increment_pipeline() - Got exception from REDIS",
+                e,
             )
             raise e
 
@@ -1931,7 +1996,7 @@ class RedisCache(BaseCache):
                     call_type=f"async_rpush <- {_get_call_stack_info()}",
                 )
             )
-            verbose_logger.error("LiteLLM Redis Cache RPUSH: - Got exception from REDIS : %s", e)
+            log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Redis Cache RPUSH: - Got exception from REDIS", e)
             raise e
 
     async def _pipeline_rpush_helper(
@@ -1999,9 +2064,11 @@ class RedisCache(BaseCache):
                     call_type=f"async_rpush_pipeline <- {_get_call_stack_info()}",
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async_rpush_pipeline() - Got exception from REDIS %s",
-                str(e),
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async_rpush_pipeline() - Got exception from REDIS",
+                e,
             )
             raise e
 
@@ -2077,7 +2144,7 @@ class RedisCache(BaseCache):
                     call_type=f"async_lpop <- {_get_call_stack_info()}",
                 )
             )
-            verbose_logger.error("LiteLLM Redis Cache LPOP: - Got exception from REDIS : %s", e)
+            log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Redis Cache LPOP: - Got exception from REDIS", e)
             raise e
 
     async def _pipeline_lpop_helper(
@@ -2188,8 +2255,10 @@ class RedisCache(BaseCache):
                     call_type=f"async_lpop_pipeline <- {_get_call_stack_info()}",
                 )
             )
-            verbose_logger.error(
-                "LiteLLM Redis Caching: async_lpop_pipeline() - Got exception from REDIS %s",
-                str(e),
+            log_redis_failure(
+                verbose_logger,
+                logging.ERROR,
+                "LiteLLM Redis Caching: async_lpop_pipeline() - Got exception from REDIS",
+                e,
             )
             raise e

litellm/compression/compress.py

@@ -205,21 +205,35 @@ def _extract_anthropic_tool_exchange_spans(
     return spans, None
 
 
+def _message_has_cache_control(message: Mapping[str, object]) -> bool:
+    if message.get("cache_control") is not None:
+        return True
+    content: Final = message.get("content")
+    if isinstance(content, list):
+        return any(isinstance(part, Mapping) and part.get("cache_control") is not None for part in content)
+    return False
+
+
 def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
     """
     Return indices of messages that must never be compressed:
     - All system messages
     - The last user message
     - The last assistant message
+    - Any message carrying an Anthropic cache_control breakpoint
 
     The last user message is what the model is being asked to act on right now,
     so compressing it replaces the live instruction with a marker. Compression
-    guardrails share this policy; see the Headroom guardrail.
+    guardrails share this policy; see the Headroom guardrail. A cache_control
+    breakpoint pins the provider's prompt-cache prefix to that row's exact
+    bytes, so rewriting a marked row anywhere in history turns the next
+    request's cache read into a cache write.
     """
     system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system")
     last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:]
-    last_assistant = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")[-1:]
-    return system_indices + last_user + last_assistant
+    assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")
+    cache_control_indices: Final = tuple(index for index, msg in enumerate(messages) if _message_has_cache_control(msg))
+    return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + cache_control_indices))
 
 
 def _combine_scores(
@@ -421,7 +435,7 @@ def compress(
         combined_scores = bm25_scores
 
     # Protected messages are never compressed
-    protected_indices: Final = get_protected_indices(normalized_messages)
+    protected_indices: Final = get_protected_indices(original_messages)
     kept_indices: set[int] = set(protected_indices)
 
     tool_exchange_spans: list[set[int]] = []

litellm/constants.py

@@ -53,6 +53,7 @@ S3_BOUNDED_OBJECT_KEY_HEAD_BYTES: Final = 64
 S3_PREFIX_DIGEST_CHARS: Final = 16
 # s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against
 MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
+MAX_FILE_LIST_LIMIT: Final = 10000
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10))
 DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1))
 budget_reservation_disabled_info_emitted = False
@@ -143,6 +144,7 @@ DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float(
     os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3)
 )
 MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = int(os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150))
+MAX_LITELLM_CALL_ID_LENGTH: Final = 256
 MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH: Final = 2048
 
 DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS: Final = 2000
@@ -225,6 +227,9 @@ PRE_CALL_EXECUTED_GUARDRAILS_KEY: Final = "_pre_call_executed_guardrails"
 # Attribute stamped on log_guardrail_information wrappers so __init_subclass__ does not wrap them again
 LOGS_GUARDRAIL_INFORMATION_MARKER: Final = "_litellm_logs_guardrail_information"
 
+# llm_provider stamped on proxy-side rate limit errors when the model resolves to no deployment
+PROXY_LLM_PROVIDER_FALLBACK: Final = "litellm_proxy"
+
 # Generic fallback for unknown models
 DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
     os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
@@ -309,6 +314,12 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float(
 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
 WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
 
+BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
+BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
+BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
+REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
+REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
+
 # SSL/TLS cipher configuration for faster handshakes
 # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
 # This balances performance with broad compatibility
@@ -459,6 +470,7 @@ REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED"
 # minimum seconds a timeout-only failure streak must span before it can open the breaker,
 # so one event-loop stall timing out many queued calls at once does not trip it
 REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0))
+REDIS_TIMEOUT_LOG_INTERVAL: Final = float(os.getenv("REDIS_TIMEOUT_LOG_INTERVAL", "5.0"))
 # Seconds of idle before a Redis cluster connection is validated with a PING and
 # reconnected if dead, so a connection silently dropped by a cluster restart
 # (e.g. ElastiCache Serverless maintenance) is not reused while broken
@@ -571,6 +583,7 @@ ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = int(
 LOGGING_WORKER_CONCURRENCY: Final = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100))  # Must be above 0
 LOGGING_WORKER_MAX_QUEUE_SIZE: Final = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000))
 LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0))
+LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS: Final = 5.0
 LOGGING_WORKER_CLEAR_PERCENTAGE: Final = int(
     os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50)
 )  # Percentage of queue to clear (default: 50%)
@@ -1963,6 +1976,8 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
 
 UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
 
+STRINGIFIED_NONE: Final[str] = "None"
+
 # A retrieved response replays the usage of the call that created it, so pricing these
 # read/management routes like inference bills the same tokens twice.
 NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset(

litellm/cost_calculator.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast
 
 from httpx import Response
 from pydantic import BaseModel
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 import litellm._logging
@@ -96,6 +97,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro
 from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
 from litellm.responses.utils import ResponseAPILoggingUtils
 from litellm.types.agents import LiteLLMSendMessageResponse
+from litellm.types.llms.base import CachedTokensDetails
 from litellm.types.llms.openai import (
     HttpxBinaryResponseContent,
     ImageGenerationRequestQuality,
@@ -310,6 +312,15 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
+OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
+
+
+class OCRPricing(TypedDict, total=False):
+    ocr_cost_per_page: ReadOnly[float | None]
+    ocr_cost_per_credit: ReadOnly[float | None]
+    annotation_cost_per_page: ReadOnly[float | None]
+
+
 def cost_per_token(
     model: str = "",
     prompt_tokens: int = 0,
@@ -344,6 +355,7 @@ def cost_per_token(
     response: Any | None = None,
     ### REQUEST MODEL ###
     request_model: str | None = None,  # original request model for router detection
+    custom_model_info: OCRPricing | None = None,
 ) -> tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -470,7 +482,8 @@ def cost_per_token(
         else:
             model_with_provider = f"{custom_llm_provider}/{model}"
         if region_name is not None:
-            model_with_provider_and_region: Final = f"{custom_llm_provider}/{region_name}/{model}"
+            bare_model: Final = model[len(_prov_prefix) :] if model_is_str and model.startswith(_prov_prefix) else model
+            model_with_provider_and_region: Final = f"{custom_llm_provider}/{region_name}/{bare_model}"
             if model_with_provider_and_region in model_cost_ref:  # use region based pricing, if it's available
                 model_with_provider = model_with_provider_and_region
     else:
@@ -558,6 +571,7 @@ def cost_per_token(
             model=model,
             custom_llm_provider=custom_llm_provider,
             response=response,
+            model_info=custom_model_info,
         )
     elif (
         call_type == "aretrieve_batch"
@@ -766,6 +780,7 @@ def _select_model_name_for_cost_calc(
     custom_pricing: bool | None = None,
     custom_llm_provider: str | None = None,
     router_model_id: str | None = None,
+    region_name: str | None = None,
 ) -> str | None:
     """
     1. If custom pricing is true, return received model name
@@ -787,8 +802,8 @@ def _select_model_name_for_cost_calc(
     provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
     explicit_pricing: Final = custom_pricing is True or base_model is not None
     priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
-    region_name: Final = (
-        _get_hidden_str_for_cost_calc(hidden_params, "region_name")
+    priced_region: Final = (
+        _get_hidden_str_for_cost_calc(hidden_params, "region_name") or region_name
         if not explicit_pricing and priced_from_response
         else None
     )
@@ -825,8 +840,10 @@ def _select_model_name_for_cost_calc(
         and custom_llm_provider is not None
         and not _model_contains_known_llm_provider(return_model)
     ):  # add provider prefix if not already present, to match model_cost
-        provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
-        return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name)
+        provider_prefix: Final = (
+            custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
+        )
+        return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
 
     return return_model
 
@@ -1288,6 +1305,7 @@ def completion_cost(
 
         service_tier = _normalize_service_tier(service_tier)
 
+        explicit_pricing: Final = custom_pricing is True or base_model is not None
         selected_model: Final = _select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
@@ -1295,6 +1313,7 @@ def completion_cost(
             custom_pricing=custom_pricing,
             base_model=base_model,
             router_model_id=router_model_id,
+            region_name=region_name,
         )
 
         potential_model_names: Final = [
@@ -1432,20 +1451,9 @@ def completion_cost(
                     )
                 elif call_type in _VIDEO_CALL_TYPES:
                     ### VIDEO GENERATION COST CALCULATION ###
-                    # Extract custom model_info for deployment-specific pricing
-                    _video_model_info: ModelInfo | None = None
-                    if custom_pricing and litellm_logging_obj is not None:
-                        _litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
-                        if _litellm_params is not None:
-                            _video_model_info = next(
-                                (
-                                    model_info
-                                    for _metadata_key in ("metadata", "litellm_metadata")
-                                    if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
-                                    is not None
-                                ),
-                                None,
-                            )
+                    _video_model_info: ModelInfo | None = _deployment_model_info(
+                        litellm_logging_obj, custom_pricing, router_model_id
+                    )
 
                     usage_obj = getattr(completion_response, "usage", None)
                     duration_seconds: float | None = None
@@ -1650,7 +1658,7 @@ def completion_cost(
                     completion_tokens=completion_tokens or 0,
                     custom_llm_provider=custom_llm_provider,
                     response_time_ms=total_time,
-                    region_name=region_name,
+                    region_name=None if explicit_pricing else region_name,
                     custom_cost_per_second=custom_cost_per_second,
                     custom_cost_per_token=custom_cost_per_token,
                     prompt_characters=prompt_characters,
@@ -1665,6 +1673,7 @@ def completion_cost(
                     data_residency=data_residency,
                     vertex_location=vertex_location,
                     response=completion_response,
+                    custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
                 )
 
                 # Get additional costs from provider (e.g., routing fees, infrastructure costs)
@@ -1859,6 +1868,7 @@ def response_cost_calculator(
     data_residency: str | None = None,  # for OpenAI regional-processing uplift (e.g. "eu", "us")
     ### VERTEX LOCATION ###
     vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
+    region_name: str | None = None,
 ) -> float:
     """
     Returns
@@ -1892,22 +1902,89 @@ def response_cost_calculator(
                 service_tier=service_tier,
                 data_residency=data_residency,
                 vertex_location=vertex_location,
+                region_name=region_name,
             )
         return response_cost
     except Exception as e:
         raise e
 
 
+def _deployment_model_info(
+    litellm_logging_obj: LitellmLoggingObject | None,
+    custom_pricing: bool | None,
+    router_model_id: str | None,
+) -> ModelInfo | None:
+    if not custom_pricing:
+        return None
+    registered_deployment_info: Final = (
+        _cost_map_model_info(router_model_id, None)
+        if router_model_id is not None and router_model_id in litellm.model_cost
+        else None
+    )
+    if registered_deployment_info is not None:
+        return registered_deployment_info
+    if litellm_logging_obj is None:
+        return None
+    litellm_params: Final = getattr(litellm_logging_obj, "litellm_params", None)
+    if litellm_params is None:
+        return None
+    return next(
+        (
+            model_info
+            for metadata_key in ("metadata", "litellm_metadata")
+            if (metadata := litellm_params.get(metadata_key)) and (model_info := metadata.get("model_info")) is not None
+        ),
+        None,
+    )
+
+
+def _ocr_model_info(
+    litellm_logging_obj: LitellmLoggingObject | None,
+    custom_pricing: bool | None,
+    router_model_id: str | None,
+) -> OCRPricing | None:
+    deployment_info: Final = _deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id)
+    litellm_params: Final = getattr(litellm_logging_obj, "litellm_params", None) if custom_pricing else None
+    if litellm_params is None:
+        return deployment_info
+    return _layered_ocr_pricing(litellm_params, deployment_info)
+
+
+def _first_ocr_price(field: OCRPricingField, *sources: Mapping[str, object] | None) -> float | None:
+    return next(
+        (price for source in sources if source is not None and isinstance(price := source.get(field), int | float)),
+        None,
+    )
+
+
+def _layered_ocr_pricing(*sources: Mapping[str, object] | None) -> OCRPricing:
+    return OCRPricing(
+        ocr_cost_per_page=_first_ocr_price("ocr_cost_per_page", *sources),
+        ocr_cost_per_credit=_first_ocr_price("ocr_cost_per_credit", *sources),
+        annotation_cost_per_page=_first_ocr_price("annotation_cost_per_page", *sources),
+    )
+
+
+def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo | None:
+    try:
+        return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
+    except Exception:
+        return None
+
+
 def ocr_cost(
     model: str,
     custom_llm_provider: str | None,
     response: object | None = None,
+    model_info: OCRPricing | None = None,
 ) -> tuple[float, float]:
     """
     Args:
         model: str - model name
         custom_llm_provider: Optional[str] - custom LLM provider
         response: Optional[Any] - response object
+        model_info: Optional[OCRPricing] - deployment-specific OCR pricing; each rate it sets
+            overrides the model cost map's, the rest fall back to the map
 
     Returns:
         Tuple[float, float]: cost of OCR processing
@@ -1925,20 +2002,15 @@ def ocr_cost(
     if response.usage_info is None:
         raise ValueError("OCR response usage_info is None")
 
-    try:
-        model_info: ModelInfo | None = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
-    except Exception:
-        model_info = None
-
     credits: Final = getattr(response.usage_info, "credits", None)
-    cost_per_credit = None
-    if model_info is not None:
-        cost_per_credit = model_info.get("ocr_cost_per_credit")
+    pricing: Final = _layered_ocr_pricing(model_info, _cost_map_model_info(model, custom_llm_provider))
+
+    cost_per_credit: Final = pricing.get("ocr_cost_per_credit")
     if credits is not None and cost_per_credit is not None:
         return cost_per_credit * credits, 0.0
 
-    ocr_cost_per_page: Final = model_info.get("ocr_cost_per_page") if model_info is not None else None
-    annotation_cost_per_page: Final = model_info.get("annotation_cost_per_page") if model_info is not None else None
+    ocr_cost_per_page: Final = pricing.get("ocr_cost_per_page")
+    annotation_cost_per_page: Final = pricing.get("annotation_cost_per_page")
     annotation_rate: Final = annotation_cost_per_page if annotation_cost_per_page is not None else ocr_cost_per_page
 
     pages_processed: Final = response.usage_info.pages_processed
@@ -2206,6 +2278,19 @@ def default_video_cost_calculator(
     return 0.0
 
 
+def _batch_rate(
+    model_info: ModelInfo,
+    key: Literal[
+        "input_cost_per_audio_token_batches",
+        "input_cost_per_image_token_batches",
+        "input_cost_per_video_token_batches",
+    ],
+    fallback: float,
+) -> float:
+    rate: Final = model_info.get(key)
+    return fallback if rate is None else rate
+
+
 def batch_cost_calculator(
     usage: Usage,
     model: str,
@@ -2265,7 +2350,29 @@ def batch_cost_calculator(
     total_prompt_cost = 0.0
     total_completion_cost = 0.0
     if input_cost_per_token_batches is not None:
-        total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
+        batch_details: Final = parse_prompt_tokens_details(usage)
+        audio_tokens, image_tokens, video_tokens = (
+            batch_details["audio_tokens"],
+            batch_details["image_tokens"],
+            batch_details["video_tokens"],
+        )
+        modality_rates: Final = (
+            _batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches),
+            _batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches),
+            _batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches),
+        )
+        total_prompt_cost = sum(
+            tokens * rate
+            for tokens, rate in zip(
+                (
+                    max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0),
+                    audio_tokens,
+                    image_tokens,
+                    video_tokens,
+                ),
+                (input_cost_per_token_batches, *modality_rates),
+            )
+        )
     elif input_cost_per_token:
         details: Final = parse_prompt_tokens_details(usage)
         cache_read_tokens: Final = details["cache_hit_tokens"]
@@ -2310,6 +2417,46 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]
     return [attr for attr in field_names if attr != "cache_creation_tokens"]
 
 
+def _combine_cached_tokens_details(
+    current: CachedTokensDetails | None, new: CachedTokensDetails
+) -> CachedTokensDetails:
+    def _sum_optional(current_value: int | None, new_value: int | None) -> int | None:
+        if current_value is None and new_value is None:
+            return None
+        return (current_value or 0) + (new_value or 0)
+
+    return CachedTokensDetails(
+        text_tokens=_sum_optional(current.text_tokens if current is not None else None, new.text_tokens),
+        audio_tokens=_sum_optional(current.audio_tokens if current is not None else None, new.audio_tokens),
+        image_tokens=_sum_optional(current.image_tokens if current is not None else None, new.image_tokens),
+    )
+
+
+def _combine_prompt_tokens_details(
+    current: PromptTokensDetailsWrapper | None, new: PromptTokensDetailsWrapper
+) -> PromptTokensDetailsWrapper:
+    base: Final = current if current is not None else PromptTokensDetailsWrapper()
+    base_values: Final = MappingProxyType(
+        {attr: getattr(base, attr) for attr in type(base).model_fields if hasattr(base, attr)}
+    )
+    summed: Final = MappingProxyType(
+        {
+            attr: (getattr(base, attr, 0) or 0) + (getattr(new, attr) or 0)
+            for attr in _summable_prompt_token_fields(new)
+            if hasattr(new, attr) and isinstance(getattr(new, attr) or 0, (int, float))
+        }
+    )
+    new_cached_tokens_details: Final = getattr(new, "cached_tokens_details", None)
+    cached_tokens_details: Final = (
+        _combine_cached_tokens_details(getattr(base, "cached_tokens_details", None), new_cached_tokens_details)
+        if isinstance(new_cached_tokens_details, CachedTokensDetails)
+        else getattr(base, "cached_tokens_details", None)
+    )
+    return PromptTokensDetailsWrapper(
+        **MappingProxyType({**base_values, **summed, "cached_tokens_details": cached_tokens_details})
+    )
+
+
 class BaseTokenUsageProcessor:
     @staticmethod
     def combine_usage_objects(usage_objects: list[Usage]) -> Usage:
@@ -2318,7 +2465,6 @@ class BaseTokenUsageProcessor:
         """
         from litellm.types.utils import (
             CompletionTokensDetailsWrapper,
-            PromptTokensDetailsWrapper,
             Usage,
         )
 
@@ -2337,27 +2483,10 @@ class BaseTokenUsageProcessor:
                         and isinstance(current_val, (int, float))
                     ):
                         setattr(combined, attr, current_val + new_val)
-            # Handle nested prompt_tokens_details
             if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
-                if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
-                    combined.prompt_tokens_details = PromptTokensDetailsWrapper()
-
-                # Check what keys exist in the model's prompt_tokens_details
-                # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
-                for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
-                    if (
-                        hasattr(usage.prompt_tokens_details, attr)
-                        and not attr.startswith("_")
-                        and not callable(_attribute_value(usage.prompt_tokens_details, attr))
-                    ):
-                        current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
-                        new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
-                        if new_val is not None and isinstance(new_val, (int, float)):
-                            setattr(
-                                combined.prompt_tokens_details,
-                                attr,
-                                current_val + new_val,
-                            )
+                combined.prompt_tokens_details = _combine_prompt_tokens_details(
+                    getattr(combined, "prompt_tokens_details", None), usage.prompt_tokens_details
+                )
 
             # Handle nested completion_tokens_details
             if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:

litellm/files/main.py

@@ -682,6 +682,10 @@ def file_list(
         )
         if provider_config is not None:
             litellm_params_dict: Final = get_litellm_params(**kwargs)
+            add_trusted_model_credentials_to_litellm_params(
+                litellm_params_dict=litellm_params_dict,
+                kwargs=kwargs,
+            )
             litellm_params_dict["api_key"] = optional_params.api_key
             litellm_params_dict["api_base"] = optional_params.api_base
 

litellm/integrations/arize/arize_phoenix_prompt_manager.py

@@ -379,7 +379,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
         """Reload prompts from Arize Phoenix."""
         if self.prompt_id:
             self._prompt_manager = None  # Reset to force reload
-            self.prompt_manager  # This will trigger reload
+            _ = self.prompt_manager  # access triggers lazy reload
 
     def should_run_prompt_management(
         self,

litellm/integrations/bitbucket/bitbucket_prompt_manager.py

@@ -406,7 +406,7 @@ class BitBucketPromptManager(CustomPromptManagement):
         """Reload prompts from BitBucket."""
         if self.prompt_id:
             self._prompt_manager = None  # Reset to force reload
-            self.prompt_manager  # This will trigger reload
+            _ = self.prompt_manager  # access triggers lazy reload
 
     def should_run_prompt_management(
         self,

litellm/integrations/custom_guardrail.py

@@ -884,7 +884,9 @@ class CustomGuardrail(CustomLogger):
         """logging_only: run apply_guardrail on copies of the logged request/response and record the verdict."""
         from litellm.llms import get_guardrail_translation_mapping
 
-        if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
+        if not self.uses_apply_guardrail_interface():
+            return kwargs, result
+        if not self._event_hook_is_event_type(GuardrailEventHooks.logging_only):
             return kwargs, result
         try:
             translation: Final = get_guardrail_translation_mapping(CallTypes(call_type))()
@@ -901,8 +903,18 @@ class CustomGuardrail(CustomLogger):
             for key, value in (litellm_params.get("metadata") or {}).items()
             if key != "standard_logging_guardrail_information"
         }
+        response: Final = (
+            kwargs.get("async_complete_streaming_response") or kwargs.get("complete_streaming_response") or result
+        )
+        from litellm.types.utils import ModelResponse
+
+        output_translation: Final = (
+            get_guardrail_translation_mapping(CallTypes.acompletion)()
+            if isinstance(response, ModelResponse)
+            else translation
+        )
         try:
-            await self._scan_logged_call(kwargs, result, translation, scratch_metadata)
+            await self._scan_logged_call(kwargs, response, translation, output_translation, scratch_metadata)
         except Exception as e:
             verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e)
         recorded: Final = scratch_metadata.get("standard_logging_guardrail_information")
@@ -919,8 +931,9 @@ class CustomGuardrail(CustomLogger):
     async def _scan_logged_call(
         self,
         kwargs: dict,  # mutable-ok: CustomLogger.async_logging_hook contract
-        result: object,
+        response: object | None,
         translation: "BaseTranslation",
+        output_translation: "BaseTranslation",
         scratch_metadata: dict,  # mutable-ok: apply_guardrail records its verdict into request metadata
     ) -> None:
         optional_params: Final = kwargs.get("optional_params") or {}
@@ -934,8 +947,10 @@ class CustomGuardrail(CustomLogger):
             "metadata": scratch_metadata,
         }
         await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
-        await translation.process_output_response(
-            response=copy.deepcopy(result), guardrail_to_apply=self, request_data=scratch_request
+        if response is None:
+            return
+        await output_translation.process_output_response(
+            response=copy.deepcopy(response), guardrail_to_apply=self, request_data=scratch_request
         )
 
     def supports_scan_only_tool_results(self) -> bool:

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -29,8 +29,6 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
     def __init__(self, bucket_name: str | None = None) -> None:
         from litellm.proxy.proxy_server import premium_user
 
-        super().__init__(bucket_name=bucket_name)
-
         self.batch_size = int(os.getenv("GCS_BATCH_SIZE", GCS_DEFAULT_BATCH_SIZE))
         self.flush_interval = int(os.getenv("GCS_FLUSH_INTERVAL", GCS_DEFAULT_FLUSH_INTERVAL_SECONDS))
         self.use_batched_logging = (
@@ -38,6 +36,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         )
         self.flush_lock = asyncio.Lock()
         super().__init__(
+            bucket_name=bucket_name,
             flush_lock=self.flush_lock,
             batch_size=self.batch_size,
             flush_interval=self.flush_interval,

litellm/integrations/otel/mappers/openinference.py

@@ -12,6 +12,7 @@ from typing import Final
 
 from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
 from litellm.integrations.otel.mappers.utils import (
+    MAX_MESSAGE_ATTRS_PER_SPAN,
     MAX_TOOL_DEFINITION_ATTRS_PER_SPAN,
     collect,
     drop_none,
@@ -26,6 +27,8 @@ from litellm.integrations.otel.model.payloads import (
     ToolDefinition,
 )
 
+_MAX_INDEXED_MESSAGES: Final = MAX_MESSAGE_ATTRS_PER_SPAN // 2
+
 
 class OpenInferenceMapper:
     """Emits OpenInference attributes for LLM_CALL spans.
@@ -84,27 +87,44 @@ class OpenInferenceMapper:
                 return {}
 
     def _llm_call(self, data: LLMCallSpanData) -> AttributeMap:
+        outputs: Final = output_messages(data)
+        indexed_in, indexed_out = self._indexed_split(len(data.messages_in), len(outputs))
         return {
             **collect(self._LLM_CALL_ATTRS, data),
             **collect(self._BLOB_ATTRS, data),
-            **self._messages("llm.input_messages", "input.value", data.messages_in),
-            **self._messages("llm.output_messages", "output.value", output_messages(data)),
+            **self._messages(
+                "llm.input_messages",
+                "input.value",
+                data.messages_in,
+                self._prompt_positions(len(data.messages_in), indexed_in),
+            ),
+            **self._messages("llm.output_messages", "output.value", outputs, range(indexed_out)),
             **self._tools(data),
         }
 
     @staticmethod
-    def _messages(prefix: str, value_key: str, messages: Sequence[object]) -> AttributeMap:
-        """Per-message ``{prefix}.{idx}.message.*`` keys + the ``value_key`` blob."""
+    def _indexed_split(inputs: int, outputs: int) -> tuple[int, int]:
+        """Prompt and response share one allowance; the response is reserved at least half of it."""
+        indexed_out: Final = min(outputs, max(_MAX_INDEXED_MESSAGES // 2, _MAX_INDEXED_MESSAGES - inputs))
+        return _MAX_INDEXED_MESSAGES - indexed_out, indexed_out
+
+    @staticmethod
+    def _prompt_positions(total: int, indexed: int) -> tuple[int, ...]:
+        """Prompt messages that get per-index attributes: message 0 and the most recent turns."""
+        if total <= indexed:
+            return tuple(range(total))
+        return (0, *range(total - indexed + 1, total))
+
+    @staticmethod
+    def _messages(prefix: str, value_key: str, messages: Sequence[object], positions: Sequence[int]) -> AttributeMap:
+        """``{prefix}.{idx}.message.*`` keys for the messages at ``positions`` + the ``value_key`` blob of all."""
         parsed: Final = [(m.get("role") if isinstance(m, dict) else None, message_content(m)) for m in messages]
         attrs: Final = drop_none(
             {
                 key: value
-                for idx, (role, content) in enumerate(parsed)
+                for idx, (role, content) in ((idx, parsed[idx]) for idx in positions)
                 for key, value in (
-                    (
-                        f"{prefix}.{idx}.message.role",
-                        role if isinstance(role, str) else None,
-                    ),
+                    (f"{prefix}.{idx}.message.role", role if isinstance(role, str) else None),
                     (f"{prefix}.{idx}.message.content", content),
                 )
             }

litellm/integrations/otel/mappers/utils.py

@@ -32,6 +32,14 @@ core telemetry no matter how many vocabularies are configured.
 """
 
 
+MAX_MESSAGE_ATTRS_PER_SPAN: Final = DEFAULT_SPAN_ATTRIBUTE_LIMIT // 8
+"""Span-wide ceiling on per-index chat message attributes, prompt and response together.
+
+An eighth is the largest share that still fits beside the tool ceiling and the core
+of every vocabulary at once. The complete conversation still rides the JSON blobs.
+"""
+
+
 def tool_attr_budget(vocabularies: int) -> int:
     """Split the span-wide tool-definition ceiling across active vocabularies."""
     return MAX_TOOL_DEFINITION_ATTRS_PER_SPAN // max(vocabularies, 1)

litellm/integrations/prometheus.py

@@ -16,6 +16,7 @@ from pydantic import BaseModel
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
+from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK
 from litellm.exceptions import (
     validate_rate_limit_category,
     validate_rate_limit_type,
@@ -2581,6 +2582,15 @@ class PrometheusLogger(CustomLogger):
             )
             return None
 
+    @staticmethod
+    def _extract_api_provider_from_exception(exception: Exception) -> str | None:
+        if not isinstance(exception, litellm.exceptions.RateLimitError):
+            return None
+        llm_provider: Final = exception.llm_provider
+        if not llm_provider or llm_provider == PROXY_LLM_PROVIDER_FALLBACK:
+            return None
+        return llm_provider
+
     async def async_post_call_failure_hook(
         self,
         request_data: dict,
@@ -2616,7 +2626,9 @@ class PrometheusLogger(CustomLogger):
             _metadata: Final = request_data.get("metadata", {}) or {}
             model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
             rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
-            api_provider: Final = self._extract_api_provider_from_request_data(request_data)
+            api_provider: Final = self._extract_api_provider_from_request_data(
+                request_data
+            ) or self._extract_api_provider_from_exception(original_exception)
             enum_values: Final = UserAPIKeyLabelValues(
                 end_user=user_api_key_dict.end_user_id,
                 user=user_api_key_dict.user_id,

litellm/integrations/weights_biases.py

@@ -4,18 +4,10 @@ imported_openAIResponse = True
 try:
     import io
     import logging
-    import sys
-    from typing import Any, TypeVar
+    from typing import Any, Literal, Protocol, TypeVar
 
     from wandb.sdk.data_types import trace_tree
 
-    if sys.version_info >= (3, 8):
-        from typing import Literal, Protocol
-    else:
-        from typing import Literal
-
-        from typing_extensions import Protocol
-
     logger: Final = logging.getLogger(__name__)
 
     K = TypeVar("K", bound=str)

litellm/litellm_core_utils/core_helpers.py

@@ -303,6 +303,16 @@ def get_metadata_variable_name_from_kwargs(
     return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
 
 
+def max_retries_per_request_hit(kwargs: Mapping[str, object], num_retries_per_request: int | None) -> bool:
+    if num_retries_per_request is None:
+        return False
+    metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs))
+    if not isinstance(metadata, Mapping):
+        return False
+    retry_count: Final = metadata.get("request_retry_count")
+    return type(retry_count) is int and 0 < retry_count and num_retries_per_request <= retry_count
+
+
 def get_or_create_metadata_bucket(
     request_data: dict,
 ) -> tuple[Literal["metadata", "litellm_metadata"], dict]:

litellm/litellm_core_utils/coroutine_checker.py

@@ -36,7 +36,7 @@ class CoroutineChecker:
         target = callback
         if not inspect.isfunction(target) and not inspect.ismethod(target):
             try:
-                call_attr: Final = getattr(target, "__call__", None)
+                call_attr: Final = getattr(target, "__call__", None)  # noqa: B004  # value unwrap so iscoroutinefunction sees through functors
                 if call_attr is not None:
                     target = call_attr
             except Exception:

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -1,6 +1,9 @@
+import inspect
 import json
 import re
 import traceback
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Any, Final, Protocol, cast
 
 import httpx
@@ -202,11 +205,17 @@ def _get_response_headers(original_exception: Exception) -> httpx.Headers | None
     return _response_headers
 
 
+def _accepted_init_kwargs(exception_class: type[Exception], candidates: Mapping[str, object]) -> Mapping[str, object]:
+    accepted: Final = inspect.signature(exception_class).parameters
+    return MappingProxyType({name: value for name, value in candidates.items() if name in accepted})
+
+
 def extract_and_raise_litellm_exception(
     response: Any | None,
     error_str: str,
     model: str,
     custom_llm_provider: str,
+    body: object | None = None,
 ):
     """
     Covers scenario where litellm sdk calling proxy.
@@ -216,32 +225,19 @@ def extract_and_raise_litellm_exception(
     Relevant Issue: https://github.com/BerriAI/litellm/issues/7259
     """
     pattern: Final = r"litellm\.\w+Error"
-
-    # Search for the exception in the error string
     match: Final = re.search(pattern, error_str)
-
-    # Extract the exception if found
-    if match:
-        exception_name = match.group(0)
-        exception_name = exception_name.strip().replace("litellm.", "")
-        raised_exception_obj: Final = getattr(litellm, exception_name, None)
-        if raised_exception_obj:
-            # Try with response parameter first, fall back to without it
-            # Some exceptions (e.g., APIConnectionError) don't accept response param
-            try:
-                raise raised_exception_obj(
-                    message=error_str,
-                    llm_provider=custom_llm_provider,
-                    model=model,
-                    response=response,
-                )
-            except TypeError:
-                # Exception doesn't accept response parameter
-                raise raised_exception_obj(
-                    message=error_str,
-                    llm_provider=custom_llm_provider,
-                    model=model,
-                )
+    if match is None:
+        return
+    exception_name: Final = match.group(0).removeprefix("litellm.")
+    raised_exception_obj: Final = getattr(litellm, exception_name, None)
+    if not raised_exception_obj:
+        return
+    raise raised_exception_obj(
+        message=error_str,
+        llm_provider=custom_llm_provider,
+        model=model,
+        **_accepted_init_kwargs(raised_exception_obj, MappingProxyType({"response": response, "body": body})),
+    )
 
 
 class _ProviderHTTPException(Protocol):
@@ -254,6 +250,23 @@ class _ProviderHTTPException(Protocol):
     llm_provider: str
 
 
+def _litellm_proxy_response(
+    original_exception: _ProviderHTTPException, custom_llm_provider: str
+) -> httpx.Response | None:
+    response: Final = getattr(original_exception, "response", None)
+    if custom_llm_provider != "litellm_proxy" or not isinstance(response, httpx.Response) or response.headers:
+        return response
+    headers: Final = getattr(original_exception, "headers", None)
+    if not isinstance(headers, Mapping) or not headers:
+        return response
+    pairs: Final = headers.multi_items() if isinstance(headers, httpx.Headers) else headers.items()
+    return httpx.Response(
+        status_code=response.status_code,
+        headers=[(str(k), str(v)) for k, v in pairs],
+        request=getattr(original_exception, "request", None),
+    )
+
+
 def _map_openai_exception(
     *,
     model: str,
@@ -264,6 +277,7 @@ def _map_openai_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
     # custom_llm_provider is openai, make it OpenAI
     message = get_error_message(error_obj=original_exception)
     if message is None:
@@ -292,14 +306,14 @@ def _map_openai_exception(
             message=f"RateLimitError: {exception_provider} - {message}",
             model=model,
             llm_provider=custom_llm_provider,
-            response=getattr(original_exception, "response", None),
+            response=response,
         )
     elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
         raise ContextWindowExceededError(
             message=f"ContextWindowExceededError: {exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif "invalid_request_error" in error_str and "model_not_found" in error_str:
@@ -307,7 +321,7 @@ def _map_openai_exception(
             message=f"{exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif "A timeout occurred" in error_str:
@@ -326,8 +340,9 @@ def _map_openai_exception(
             message=f"ContentPolicyViolationError: {exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
+            body=getattr(original_exception, "body", None),
         )
     elif "invalid_encrypted_content" in error_str or "could not be verified" in error_str:
         helpful_message: Final = (
@@ -345,7 +360,7 @@ def _map_openai_exception(
             message=helpful_message,
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
             body=getattr(original_exception, "body", None),
         )
@@ -354,7 +369,7 @@ def _map_openai_exception(
             message=f"{exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
             body=getattr(original_exception, "body", None),
         )
@@ -372,7 +387,7 @@ def _map_openai_exception(
             message=f"RateLimitError: {exception_provider} - {message}",
             model=model,
             llm_provider=custom_llm_provider,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif (
@@ -383,7 +398,7 @@ def _map_openai_exception(
             message=f"AuthenticationError: {exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif "Mistral API raised a streaming error" in error_str:
@@ -402,15 +417,16 @@ def _map_openai_exception(
                 message=f"{exception_provider} - {message}",
                 llm_provider=custom_llm_provider,
                 model=model,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
+                body=getattr(original_exception, "body", None),
             )
         elif original_exception.status_code == 401:
             raise AuthenticationError(
                 message=f"AuthenticationError: {exception_provider} - {message}",
                 llm_provider=custom_llm_provider,
                 model=model,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 404:
@@ -418,7 +434,7 @@ def _map_openai_exception(
                 message=f"NotFoundError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 408:
@@ -433,7 +449,7 @@ def _map_openai_exception(
                 message=f"{exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
                 body=getattr(original_exception, "body", None),
             )
@@ -442,7 +458,7 @@ def _map_openai_exception(
                 message=f"RateLimitError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 500:
@@ -450,7 +466,7 @@ def _map_openai_exception(
                 message=f"InternalServerError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 502:
@@ -458,7 +474,7 @@ def _map_openai_exception(
                 message=f"BadGatewayError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 503:
@@ -466,7 +482,7 @@ def _map_openai_exception(
                 message=f"ServiceUnavailableError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 504:  # gateway timeout error
@@ -2423,10 +2439,11 @@ def exception_type(
                 custom_llm_provider == "litellm_proxy"
             ):  # handle special case where calling litellm proxy + exception str contains error message
                 extract_and_raise_litellm_exception(
-                    response=getattr(original_exception, "response", None),
+                    response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
                     error_str=error_str,
                     model=model,
                     custom_llm_provider=custom_llm_provider,
+                    body=getattr(original_exception, "body", None),
                 )
             if (
                 custom_llm_provider == "openai"

litellm/litellm_core_utils/fallback_generalizations.py

@@ -34,6 +34,11 @@ rules never mix the two and never use ``extends``. A rule whose
 Rules are only consulted after exact and case-insensitive lookups miss, so an
 exact cost-map entry always takes precedence over any rule.
 
+Rules flagged with ``fill_missing_for_providers: [..]`` also fill only keys
+missing from an exact cost-map entry when the entry's ``litellm_provider`` is
+listed, while values already present on the entry win on conflict. Only flagged
+capability rules participate in this fill; routing rules never do.
+
 Patterns are matched case-insensitively with ``re.search`` and are not implicitly
 anchored: a rule must include ``^`` and ``$`` to bind to the whole model name,
 otherwise it matches as a substring. Keeping anchoring in the regex makes the rule
@@ -46,17 +51,19 @@ Rules are compiled and classified once, at install time. The match functions are
 O(number of rules); callers must only invoke them on a cache miss.
 """
 
+import logging
 import re
+from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import Final
 
-from litellm._logging import verbose_logger
-
+verbose_logger: Final = logging.getLogger("LiteLLM")
 NAME_FIELD: Final = "name"
 PATTERN_FIELD: Final = "pattern"
 MODEL_INFO_FIELD: Final = "model_info"
 PROVIDER_KEY: Final = "litellm_provider"
 LEGACY_EXTENDS_FIELD: Final = "extends"
+FILL_MISSING_FOR_PROVIDERS_FIELD: Final = "fill_missing_for_providers"
 
 
 def _resolve_legacy_extends(rules: list) -> list:
@@ -98,11 +105,28 @@ class _RoutingRule:
 class _CapabilityRule:
     pattern: re.Pattern
     model_info: dict
+    fill_missing_for_providers: frozenset[str]
 
 
 _CompiledRule = _RoutingRule | _CapabilityRule
 
 
+def _parse_fill_missing_for_providers(rule: Mapping[str, object], pattern_label: object) -> frozenset[str] | None:
+    if FILL_MISSING_FOR_PROVIDERS_FIELD not in rule:
+        return frozenset()
+    raw_fill_missing_for_providers: Final = rule.get(FILL_MISSING_FOR_PROVIDERS_FIELD)
+    if not isinstance(raw_fill_missing_for_providers, (list, tuple)) or not all(
+        isinstance(provider, str) for provider in raw_fill_missing_for_providers
+    ):
+        verbose_logger.warning(
+            "LiteLLM: skipping malformed fallback generalization rule %s ('%s' must be a list of provider strings).",
+            rule.get(NAME_FIELD, pattern_label),
+            FILL_MISSING_FOR_PROVIDERS_FIELD,
+        )
+        return None
+    return frozenset(raw_fill_missing_for_providers)
+
+
 def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:
     if not isinstance(rule, dict):
         return ()
@@ -125,8 +149,17 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:
             e,
         )
         return ()
+    fill_missing_for_providers: Final = _parse_fill_missing_for_providers(rule, pattern)
+    if fill_missing_for_providers is None:
+        return ()
     if PROVIDER_KEY not in model_info:
-        return (_CapabilityRule(pattern=compiled, model_info=model_info),)
+        return (
+            _CapabilityRule(
+                pattern=compiled,
+                model_info=model_info,
+                fill_missing_for_providers=fill_missing_for_providers,
+            ),
+        )
     provider: Final = model_info[PROVIDER_KEY]
     if not isinstance(provider, str):
         verbose_logger.warning(
@@ -140,7 +173,11 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:
         return (_RoutingRule(pattern=compiled, provider=provider),)
     return (
         _RoutingRule(pattern=compiled, provider=provider),
-        _CapabilityRule(pattern=compiled, model_info=model_info),
+        _CapabilityRule(
+            pattern=compiled,
+            model_info=model_info,
+            fill_missing_for_providers=fill_missing_for_providers,
+        ),
     )
 
 
@@ -151,6 +188,7 @@ class _FallbackGeneralizations:
         self.rules: list = []
         self.routing_rules: tuple = ()
         self.capability_rules: tuple = ()
+        self.fill_missing_rules: tuple[_CapabilityRule, ...] = ()
 
     def set_rules(self, rules: list | None) -> None:
         installed: Final = rules if isinstance(rules, list) else []
@@ -158,6 +196,7 @@ class _FallbackGeneralizations:
         self.rules = installed
         self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule))
         self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule))
+        self.fill_missing_rules = tuple(rule for rule in self.capability_rules if rule.fill_missing_for_providers)
 
     def match_routing(self, model: str) -> str | None:
         if not model:
@@ -175,6 +214,21 @@ class _FallbackGeneralizations:
             return None
         return {key: value for model_info in matched for key, value in model_info.items()}
 
+    def match_fill_missing(self, model: str, provider: str) -> Mapping[str, object] | None:
+        if not model or not provider:
+            return None
+        matched = tuple(
+            rule.model_info
+            for rule in self.fill_missing_rules
+            if provider in rule.fill_missing_for_providers and rule.pattern.search(model) is not None
+        )
+        if not matched:
+            return None
+        fill_missing: Final[Mapping[str, object]] = {
+            key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY
+        }
+        return fill_missing or None
+
 
 _registry: Final = _FallbackGeneralizations()
 
@@ -210,3 +264,14 @@ def match_capability_generalizations(model: str) -> dict | None:
     capability rule matches. O(number of rules); only call once exact lookups have missed.
     """
     return _registry.match_capabilities(model)
+
+
+def match_fill_missing_generalizations(model: str, provider: str) -> Mapping[str, object] | None:
+    """Return flagged capability rules matching ``model`` for ``provider``.
+
+    Later rules override earlier ones on key conflicts. Only rules listing
+    ``provider`` in ``fill_missing_for_providers`` contribute. Returns ``None``
+    when no flagged rule matches. O(number of rules); only call once exact
+    lookups have matched.
+    """
+    return _registry.match_fill_missing(model, provider)

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -601,12 +601,15 @@ def _get_openai_compatible_provider_info(
             dynamic_api_key,
         ) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "bedrock_mantle":
+        from litellm.llms.bedrock_mantle.common_utils import split_mantle_region_prefix
+
         (
             api_base,
             dynamic_api_key,
         ) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
             api_base, api_key, litellm_params=litellm_params, model=model
         )
+        model = split_mantle_region_prefix(model)[1]  # rebind-ok: the prefix is routing only, not a Mantle model id
     elif custom_llm_provider == "nvidia_nim":
         # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
         api_base = api_base or get_secret("NVIDIA_NIM_API_BASE") or "https://integrate.api.nvidia.com/v1"

litellm/litellm_core_utils/litellm_logging.py

@@ -454,6 +454,17 @@ def _resolve_vertex_location_for_cost(
     return VertexBase.get_vertex_region(configured_location, model)
 
 
+def _resolve_mantle_region_for_cost(
+    custom_llm_provider: str | None,
+    litellm_params: Mapping[str, object] | None,
+) -> str | None:
+    if custom_llm_provider != "bedrock_mantle":
+        return None
+    from litellm.llms.bedrock_mantle.common_utils import resolve_mantle_region
+
+    return resolve_mantle_region(litellm_params or MappingProxyType({}))
+
+
 def _provider_response_id(source: object) -> str | None:
     candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
     return candidate if isinstance(candidate, str) and candidate else None
@@ -545,7 +556,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # ids leaking into a different, later request on the same thread. Sync
         # support is deferred to a follow-up PR with its own safe-restore
         # mechanism; async calls (the proxy's only call path) are unaffected.
-        if supports_correlation_logging:
+        if supports_correlation_logging and litellm.request_correlation_in_logs:
             set_trace_id(self.litellm_trace_id)
             set_session_id(self.litellm_session_id)
         # set_trace_id()/set_session_id() sanitize (strip control chars, bound
@@ -1768,6 +1779,10 @@ class Logging(LiteLLMLoggingBaseClass):
                     optional_params=self.optional_params,
                     model=litellm_model_name or self.model,
                 ),
+                "region_name": _resolve_mantle_region_for_cost(
+                    custom_llm_provider=self.model_call_details.get("custom_llm_provider", None),
+                    litellm_params=self.model_call_details.get("litellm_params"),
+                ),
             }
         except Exception as e:  # error creating kwargs for cost calculation
             debug_info = StandardLoggingModelCostFailureDebugInformation(
@@ -2427,7 +2442,7 @@ class Logging(LiteLLMLoggingBaseClass):
         call) would leave the outer request's subsequent log lines stamped with
         the nested call's trace_id/session_id instead of its own.
 
-        Uses a plain set() of the captured pre-call value rather than
+        Uses a plain contextvar set() of the captured pre-call value rather than
         contextvars.Token-based reset(), since this can end up called from a
         different asyncio Task/context than __init__ ran in (e.g. the request
         task's own wrapper() finally block, plus async_success_handler
@@ -2438,8 +2453,8 @@ class Logging(LiteLLMLoggingBaseClass):
         that Task's view of the contextvars, so calling it multiple times
         (once per Task involved in this attempt) is required, not just safe.
         """
-        set_trace_id(self._pre_call_trace_id)
-        set_session_id(self._pre_call_session_id)
+        trace_id_var.set(self._pre_call_trace_id)
+        session_id_var.set(self._pre_call_session_id)
 
     def _restore_correlation_context_if_unclaimed(self) -> None:
         """Guarded variant for __del__-triggered cleanup only.

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -9,6 +9,8 @@ from types import MappingProxyType
 from typing import Any, Final, Literal, TypedDict, cast
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
+from typing_extensions import ReadOnly
+
 import litellm
 from litellm._internal_context import current_billing_time
 from litellm._logging import verbose_logger
@@ -482,11 +484,12 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
 def _apply_off_peak_to_base_costs(
     model_info: ModelInfo,
     current_time: datetime | None,
-    base_costs: tuple[float, float, float, float, float],
+    base_costs: tuple[float, float, float, float | None, float],
 ) -> tuple[float, float, float, float, float]:
     """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
-    produced them. The one-hour cache-creation rate passes through untouched, since
-    off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
+    produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
+    present one passes through untouched and an absent one resolves to the applied
+    cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
     """
     prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
     rates: Final = apply_off_peak_pricing(
@@ -504,7 +507,7 @@ def _apply_off_peak_to_base_costs(
         rates.input_rate,
         rates.output_rate,
         rates.cache_creation_rate,
-        cache_creation_above_1hr,
+        rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
         rates.cache_read_rate,
     )
 
@@ -530,6 +533,11 @@ def _get_token_base_cost(
     `missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved
     input rate instead of 0.0; an explicit 0.0 rate stays a real price either way.
 
+    An absent cache-creation rate always resolves to the resolved input rate, the way the
+    tiered table and custom deployment pricing already do, since a provider that publishes
+    no write price bills cache writes as ordinary input. An absent 1h write rate resolves
+    to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both.
+
     Returns:
         Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
     """
@@ -552,10 +560,9 @@ def _get_token_base_cost(
         output_image_cost: Final = _get_cost_per_unit(model_info, "output_cost_per_image_token", None)
         if output_image_cost is not None:
             completion_base_cost = cast(float, output_image_cost)
-    cache_creation_cost = cast(float, _get_cost_per_unit(model_info, cache_creation_cost_key))
-    cache_creation_cost_above_1hr = cast(
-        float,
-        _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
+    cache_creation_cost = _get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
+    cache_creation_cost_above_1hr = _get_cost_per_unit(
+        model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
     )
     cache_read_cost = _get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
 
@@ -637,22 +644,10 @@ def _get_token_base_cost(
                         else f"cache_read_input_token_cost_above_{threshold_str}_tokens"
                     )
 
-                    cache_creation_cost = cast(
-                        float,
-                        _get_cost_per_unit(
-                            model_info,
-                            cache_creation_tiered_key,
-                            cache_creation_cost,
-                        ),
-                    )
+                    cache_creation_cost = _get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
 
-                    cache_creation_cost_above_1hr = cast(
-                        float,
-                        _get_cost_per_unit(
-                            model_info,
-                            cache_creation_1hr_tiered_key,
-                            cache_creation_cost_above_1hr,
-                        ),
+                    cache_creation_cost_above_1hr = _get_cost_per_unit(
+                        model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
                     )
 
                     cache_read_cost = _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
@@ -663,16 +658,16 @@ def _get_token_base_cost(
             except Exception:
                 continue
 
+    input_rate_for_missing_cache_rates: Final = _off_peak_rate(
+        _open_off_peak_block(model_info, current_time) or MappingProxyType({}),
+        "input_cost_per_token",
+        prompt_base_cost,
+    )
     if cache_read_cost is None:
-        cache_read_cost = (
-            _off_peak_rate(
-                _open_off_peak_block(model_info, current_time) or MappingProxyType({}),
-                "input_cost_per_token",
-                prompt_base_cost,
-            )
-            if missing_cache_read_uses_input
-            else 0.0
-        )
+        cache_read_cost = input_rate_for_missing_cache_rates if missing_cache_read_uses_input else 0.0
+    resolved_cache_creation_cost: Final = (
+        input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
+    )
 
     return _apply_off_peak_to_base_costs(
         model_info,
@@ -680,7 +675,7 @@ def _get_token_base_cost(
         (
             prompt_base_cost,
             completion_base_cost,
-            cache_creation_cost,
+            resolved_cache_creation_cost,
             cache_creation_cost_above_1hr,
             cache_read_cost,
         ),
@@ -772,6 +767,7 @@ def calculate_cache_writing_cost(
 
 class PromptTokensDetailsResult(TypedDict):
     cache_hit_tokens: int
+    cache_hit_audio_tokens: ReadOnly[int]
     cache_creation_tokens: int
     cache_creation_token_details: CacheCreationTokenDetails | None
     text_tokens: int
@@ -802,12 +798,34 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
         )
         or None
     )
-    text_tokens: Final = (
-        cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
-        or 0  # default to prompt tokens, if this field is not set
+    cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
+    cached_audio_tokens: Final = min(
+        _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens
+    )
+    cached_text_tokens: Final = min(
+        _get_token_detail_value(cached_tokens_details, "text_tokens") or 0,
+        cache_hit_tokens - cached_audio_tokens,
+    )
+    cached_image_tokens: Final = min(
+        _get_token_detail_value(cached_tokens_details, "image_tokens") or 0,
+        cache_hit_tokens - cached_audio_tokens - cached_text_tokens,
+    )
+    text_tokens: Final = max(
+        (
+            cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
+            or 0  # default to prompt tokens, if this field is not set
+        )
+        - cached_text_tokens,
+        0,
+    )
+    audio_tokens: Final = max(
+        (cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens,
+        0,
+    )
+    image_tokens: Final = max(
+        (cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens,
+        0,
     )
-    audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0
-    image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0
     video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0))
     character_count: Final = (
         cast(
@@ -835,6 +853,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
 
     return PromptTokensDetailsResult(
         cache_hit_tokens=cache_hit_tokens,
+        cache_hit_audio_tokens=cached_audio_tokens,
         cache_creation_tokens=cache_creation_tokens,
         cache_creation_token_details=cache_creation_token_details,
         text_tokens=text_tokens,
@@ -918,15 +937,28 @@ def _calculate_input_cost(
     prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
 
     ### CACHE READ COST - Now uses tiered pricing
-    prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
+    cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
+    audio_cache_read_rate: Final = _get_cost_per_unit(
+        model_info,
+        _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
+        None,
+    )
+    prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
+    prompt_cost += float(cache_hit_audio_tokens) * (
+        audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
+    )
 
     ### AUDIO COST
-    if prompt_tokens_details["audio_tokens"]:
+    if prompt_tokens_details["audio_tokens"] and not (
+        prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
+    ):
         audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
         prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
 
     ### IMAGE TOKEN COST
-    if prompt_tokens_details["image_tokens"]:
+    if prompt_tokens_details["image_tokens"] and not (
+        prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
+    ):
         # For image token costs:
         # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
         image_token_cost_key = "input_cost_per_image_token"
@@ -935,7 +967,9 @@ def _calculate_input_cost(
         prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
 
     ### VIDEO TOKEN COST
-    if prompt_tokens_details["video_tokens"]:
+    if prompt_tokens_details["video_tokens"] and not (
+        prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
+    ):
         video_token_cost_key = "input_cost_per_video_token"
         if model_info.get(video_token_cost_key) is None:
             video_token_cost_key = "input_cost_per_token"
@@ -1149,6 +1183,7 @@ def generic_cost_per_token(
     ### PROCESSING COST
     prompt_tokens_details = PromptTokensDetailsResult(
         cache_hit_tokens=0,
+        cache_hit_audio_tokens=0,
         cache_creation_tokens=0,
         cache_creation_token_details=None,
         text_tokens=usage.prompt_tokens,
@@ -1319,6 +1354,7 @@ class BilledTokenRates:
     input_cost_per_token: float
     output_cost_per_token: float
     cache_read_input_token_cost: float
+    cache_read_input_audio_token_cost: float
     cache_creation_input_token_cost: float
     cache_creation_input_token_cost_above_1hr: float
     output_cost_per_reasoning_token: float
@@ -1330,6 +1366,7 @@ class BilledTokenRates:
             input_cost_per_token=self.input_cost_per_token * multiplier,
             output_cost_per_token=self.output_cost_per_token * multiplier,
             cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
+            cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
             cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
             cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
             output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
@@ -1353,15 +1390,16 @@ def _reasoning_token_count(usage: Usage) -> int:
     return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
 
 
-def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]:
-    """(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details
-    first, then the private top-level counters the Usage constructor mirrors cache tokens onto for
-    providers/callers that bypass the details."""
+def _cache_token_counts(usage: Usage) -> tuple[int, int, int, CacheCreationTokenDetails | None]:
+    """(cache read tokens, cached audio tokens, cache creation tokens, cache creation details): read from
+    prompt_tokens_details first, then the private top-level counters the Usage constructor mirrors cache
+    tokens onto for providers/callers that bypass the details."""
     parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None
     parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0
     parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0
     return (
         parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)),
+        parsed["cache_hit_audio_tokens"] if parsed is not None else 0,
         parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)),
         parsed["cache_creation_token_details"] if parsed is not None else None,
     )
@@ -1372,11 +1410,13 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
     cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does."""
     input_rate: Final = custom_cost_per_token["input_cost_per_token"]
     output_rate: Final = custom_cost_per_token["output_cost_per_token"]
+    cache_read_rate: Final = custom_cost_per_token.get("cache_read_input_token_cost", input_rate)
     cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate)
     return BilledTokenRates(
         input_cost_per_token=input_rate,
         output_cost_per_token=output_rate,
-        cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
+        cache_read_input_token_cost=cache_read_rate,
+        cache_read_input_audio_token_cost=cache_read_rate,
         cache_creation_input_token_cost=cache_creation_rate,
         cache_creation_input_token_cost_above_1hr=cache_creation_rate,
         output_cost_per_reasoning_token=output_rate,
@@ -1413,6 +1453,11 @@ def _cost_map_billed_rates(
         completion_base_cost=completion_base_cost,
         current_time=billing_time,
     )
+    audio_cache_read_rate: Final = _get_cost_per_unit(
+        model_info,
+        _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
+        None,
+    )
     multiplier: Final = (
         _get_regional_uplift_multiplier(model_info, data_residency)
         * get_vertex_regional_endpoint_uplift(model_info, vertex_location)
@@ -1422,6 +1467,9 @@ def _cost_map_billed_rates(
         input_cost_per_token=prompt_base_cost,
         output_cost_per_token=completion_base_cost,
         cache_read_input_token_cost=cache_read_cost_rate,
+        cache_read_input_audio_token_cost=(
+            audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate
+        ),
         cache_creation_input_token_cost=cache_creation_cost_rate,
         cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
         output_cost_per_reasoning_token=reasoning_rate,
@@ -1494,7 +1542,9 @@ def get_token_type_cost_breakdown(
     if rates is None:
         return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
 
-    cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage)
+    cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
+        usage
+    )
     cache_creation_cost: Final = (
         float(cache_creation_tokens) * rates.cache_creation_input_token_cost
         if custom_cost_per_token is not None
@@ -1507,7 +1557,10 @@ def get_token_type_cost_breakdown(
     )
     return TokenTypeCostBreakdown(
         reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
-        cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost,
+        cache_read_cost=(
+            float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
+            + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
+        ),
         cache_creation_cost=cache_creation_cost,
         rates=rates,
     )

litellm/litellm_core_utils/logging_worker.py

@@ -18,11 +18,16 @@ from litellm.constants import (
     LOGGING_WORKER_CONCURRENCY,
     LOGGING_WORKER_MAX_QUEUE_SIZE,
     LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
+    LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS,
     MAX_ITERATIONS_TO_CLEAR_QUEUE,
     MAX_TIME_TO_CLEAR_QUEUE,
 )
 
 
+def _coroutine_name(coroutine: Coroutine) -> str:
+    return getattr(coroutine, "__qualname__", None) or getattr(coroutine, "__name__", None) or type(coroutine).__name__
+
+
 class LoggingTask(TypedDict):
     """
     A logging task with its associated context to ensure logging is executed in
@@ -47,10 +52,12 @@ class LoggingWorker:
         timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
         max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
         concurrency: int = LOGGING_WORKER_CONCURRENCY,
+        timeout_summary_window: float = LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS,
     ):
         self.timeout = timeout
         self.max_queue_size = max_queue_size
         self.concurrency = concurrency
+        self.timeout_summary_window = timeout_summary_window
         self._queue: asyncio.Queue[LoggingTask] | None = None
         self._worker_task: asyncio.Task | None = None
         self._running_tasks: set[asyncio.Task] = set()
@@ -59,6 +66,10 @@ class LoggingWorker:
         self._bound_loop: asyncio.AbstractEventLoop | None = None
         self._last_aggressive_clear_time: float = 0.0
         self._aggressive_clear_in_progress: bool = False
+        self._timeout_total: int = 0
+        self._timeout_burst_count: int = 0
+        self._timeout_last_callback: str | None = None
+        self._timeout_summary_task: asyncio.Task | None = None
 
         # Register cleanup handler to flush remaining events on exit
         atexit.register(self._flush_on_exit)
@@ -136,6 +147,8 @@ class LoggingWorker:
             self._sem = None
             self._worker_task = None
             self._running_tasks.clear()
+            self._timeout_summary_task = None
+            self._timeout_burst_count = 0
             self._queue = new_queue
             self._bound_loop = current_loop
             return
@@ -156,12 +169,15 @@ class LoggingWorker:
         """Runs the logging task and handles cleanup. Releases semaphore when done."""
         try:
             if self._queue is not None:
+                # Run the coroutine in its original context
+                callback_task: Final = task["context"].run(asyncio.create_task, task["coroutine"])
                 try:
-                    # Run the coroutine in its original context
-                    await asyncio.wait_for(
-                        task["context"].run(asyncio.create_task, task["coroutine"]),
-                        timeout=self.timeout,
-                    )
+                    await asyncio.wait_for(callback_task, timeout=self.timeout)
+                except asyncio.TimeoutError as e:
+                    if callback_task.cancelled():
+                        self._record_callback_timeout(task["coroutine"])
+                    else:
+                        verbose_logger.exception("LoggingWorker error: %s", e)
                 except Exception as e:
                     verbose_logger.exception("LoggingWorker error: %s", e)
                 finally:
@@ -171,6 +187,35 @@ class LoggingWorker:
             # Always release semaphore, even if queue is None
             sem.release()
 
+    def _record_callback_timeout(self, coroutine: Coroutine) -> None:
+        """Count a callback timeout and arm a debounced summary, so a burst of timeouts
+        (e.g. a slow Redis timing out many callbacks at once) logs one bounded line rather
+        than a full ERROR stacktrace per callback."""
+        self._timeout_total += 1
+        self._timeout_burst_count += 1
+        self._timeout_last_callback = _coroutine_name(coroutine)
+        if self._timeout_summary_task is None or self._timeout_summary_task.done():
+            self._timeout_summary_task = asyncio.create_task(self._flush_timeout_summary())
+
+    async def _flush_timeout_summary(self) -> None:
+        """After the burst settles, log one bounded summary covering every timeout in it."""
+        await asyncio.sleep(self.timeout_summary_window)
+        self._emit_timeout_summary()
+
+    def _emit_timeout_summary(self) -> None:
+        """Log one bounded summary for the current burst and reset the burst counter."""
+        burst_count: Final = self._timeout_burst_count
+        self._timeout_burst_count = 0
+        if burst_count <= 0:
+            return
+        verbose_logger.warning(
+            "LoggingWorker: %d callback(s) timed out after %ss (callback: %s); %d timed out since start",
+            burst_count,
+            self.timeout,
+            self._timeout_last_callback,
+            self._timeout_total,
+        )
+
     async def _worker_loop(self) -> None:
         """Main worker loop that gets tasks and schedules them to run concurrently."""
         try:
@@ -406,6 +451,11 @@ class LoggingWorker:
 
     async def stop(self) -> None:
         """Stop the logging worker and clean up resources."""
+        if self._timeout_summary_task is not None:
+            self._timeout_summary_task.cancel()
+            self._timeout_summary_task = None
+        self._emit_timeout_summary()
+
         if self._worker_task is None and not self._running_tasks:
             # No worker launched and no in-flight tasks to drain.
             return

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1757,7 +1757,7 @@ def convert_to_anthropic_tool_invoke(
     anthropic_tool_invoke: Final[list[AnthropicMessagesToolUseParam | dict[str, object]]] = []
 
     for tool in tool_calls:
-        if not get_attribute_or_key(tool, "type") == "function":
+        if get_attribute_or_key(tool, "type") != "function":
             continue
 
         tool_id = cast(str, get_attribute_or_key(tool, "id"))

litellm/litellm_core_utils/realtime_streaming.py

@@ -10,6 +10,7 @@ from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import redact_internal_details_from_client_message, verbose_logger
+from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
 from litellm.types.llms.openai import (
@@ -35,9 +36,6 @@ else:
     CLIENT_CONNECTION_CLASS = Any
 
 
-REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
-
-
 @dataclass(frozen=True, slots=True)
 class BackendClose:
     code: int
@@ -445,6 +443,12 @@ class RealTimeStreaming:
             )
             sent = False
             for msg in transformed:
+                if isinstance(msg, bytes):
+                    await self.provider_config.pace_backend_send(msg)
+                    await self.backend_ws.send(msg)
+                    self._content_sent_after_setup = True
+                    sent = True
+                    continue
                 try:
                     msg_obj = _decode_json_object(msg)
                 except (json.JSONDecodeError, TypeError):
@@ -1013,7 +1017,7 @@ class RealTimeStreaming:
                     cast(str, transcript),
                     item_id=cast(str | None, event.get("item_id")),
                 )
-                if not blocked:
+                if not blocked and not self._is_transcription_session:
                     await self._send_to_backend(json.dumps({"type": "response.create"}))
                 continue
             ## LOGGING
@@ -1147,6 +1151,7 @@ class RealTimeStreaming:
         self._logging_worker.ensure_initialized_and_enqueue(
             self.logging_obj.dispatch_failure_handlers(error, traceback.format_exc(), prefer_async_handlers=True)
         )
+        self.logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True
 
     @staticmethod
     def _detect_beta_header(websocket: ScopedWebSocket) -> bool:

litellm/litellm_core_utils/token_counter.py

@@ -454,7 +454,7 @@ def token_counter(
         params: Final = _MessageCountParams(model, custom_tokenizer)
         num_tokens = _count_messages(params, new_messages, use_default_image_token_count, default_token_count)
         if count_response_tokens is False:
-            includes_system_message: Final = any([message.get("role", None) == "system" for message in new_messages])
+            includes_system_message: Final = any(message.get("role", None) == "system" for message in new_messages)
             num_tokens += _count_extra(params.count_function, tools, tool_choice, includes_system_message)
 
     else:

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -20,6 +20,7 @@ from itertools import chain, repeat
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
 
+from pydantic import TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm._logging import verbose_proxy_logger
@@ -44,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     scoped_structured_message_indices,
     stream_item_field,
     stream_item_fingerprint,
+    unappliable_request_rewrite,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
     AnthropicPassthroughLoggingHandler,
@@ -103,9 +105,24 @@ class ToolResultBlockTextTarget:
     block_idx: int
 
 
-InputWriteBackTarget = (
-    MessageContentTarget | ContentBlockTextTarget | ToolResultStringTarget | ToolResultBlockTextTarget
-)
+@dataclass(frozen=True, slots=True)
+class SystemStringTarget:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class SystemBlockTextTarget:
+    block_idx: int
+
+
+@dataclass(frozen=True, slots=True)
+class ToolUseInputTarget:
+    msg_idx: int
+    content_idx: int
+
+
+MessageTextTarget = MessageContentTarget | ContentBlockTextTarget | ToolResultStringTarget | ToolResultBlockTextTarget
+InputWriteBackTarget = SystemStringTarget | SystemBlockTextTarget | MessageTextTarget
 
 
 def _as_str_mapping(value: Mapping[str, object]) -> Mapping[str, object]:
@@ -146,10 +163,17 @@ class ScannedText:
     target: InputWriteBackTarget
 
 
+@dataclass(frozen=True, slots=True)
+class ScannedToolCall:
+    tool_call: ChatCompletionToolCallChunk
+    target: ToolUseInputTarget
+
+
 @dataclass(frozen=True, slots=True)
 class ExtractedInput:
     scanned: tuple[ScannedText, ...]
     images: tuple[str, ...]
+    tool_calls: tuple[ScannedToolCall, ...] = ()
 
 
 EMPTY_EXTRACTED_INPUT: Final = ExtractedInput(scanned=(), images=())
@@ -161,6 +185,74 @@ class _ToolCallShape:
     arguments: str
 
 
+def _is_client_tool_use(block: Mapping[str, object]) -> bool:
+    return (
+        block.get("type") == "tool_use"
+        and isinstance(block.get("id"), str)
+        and isinstance(block.get("name"), str)
+        and isinstance(block.get("input"), dict)
+    )
+
+
+def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
+    if not isinstance(system, list):
+        return
+    text_blocks: Final = tuple(block for block in system if isinstance(block, dict) and block.get("type") == "text")
+    if block_idx < len(text_blocks):
+        text_blocks[block_idx]["text"] = (
+            response  # mutable-ok: guardrails rewrite the caller's request payload in place
+        )
+
+
+def _write_back_message_text(message: _WritableMessage, target: MessageTextTarget, response: str) -> None:
+    content: Final = message.get("content", None)
+    if content is None:
+        return
+    match target:
+        case MessageContentTarget():
+            if isinstance(content, str):
+                message["content"] = response  # mutable-ok: guardrails rewrite the caller's request payload in place
+        case ContentBlockTextTarget(content_idx=content_idx):
+            if isinstance(content, list):
+                content[content_idx]["text"] = (
+                    response  # mutable-ok: guardrails rewrite the caller's request payload in place
+                )
+        case ToolResultStringTarget(content_idx=content_idx):
+            if isinstance(content, list):
+                content[content_idx]["content"] = (
+                    response  # mutable-ok: guardrails rewrite the caller's request payload in place
+                )
+        case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
+            if isinstance(content, list):
+                content[content_idx]["content"][block_idx]["text"] = (
+                    response  # mutable-ok: guardrails rewrite the caller's request payload in place
+                )
+        case _:
+            assert_never(target)
+
+
+_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+
+def _rewritten_tool_use_input(arguments: str) -> Mapping[str, object] | None:
+    try:
+        return _TOOL_USE_INPUT_ADAPTER.validate_json(arguments)
+    except ValidationError:
+        return None
+
+
+def _write_back_tool_use(
+    message: _WritableMessage, target: ToolUseInputTarget, shape: _ToolCallShape, rewritten_input: Mapping[str, object]
+) -> None:
+    content: Final = message.get("content", None)
+    block: Final = content[target.content_idx] if isinstance(content, list) else None
+    if not isinstance(block, dict):
+        return
+    block["input"] = rewritten_input  # mutable-ok: guardrails rewrite the caller's request payload in place
+    if shape.name is not None and shape.name != block.get("name"):
+        block["name"] = shape.name  # mutable-ok: guardrails rewrite the caller's request payload in place
+
+
 @dataclass(frozen=True, slots=True)
 class _SSEFieldRewrite:
     """One field of one nested section of a buffered SSE event, rewritten."""
@@ -452,9 +544,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
         scan_only_tool_results: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
 
-        # Exclude only the trusted top-level prompt. In-sequence system entries are untrusted
-        # and must stay aligned with texts_to_check for positional masking. When the top-level
-        # prompt is included, the pre-existing count mismatch disables positional masking.
+        # The top-level prompt is translated on its own below so it can be hoisted in front of
+        # any mid-turn system entries and scanned first, aligned with that structured position.
         translation_source: Final = {  # mutable-ok: API message payload
             key: value for key, value in data.items() if key != "system"
         }
@@ -490,7 +581,12 @@ class AnthropicMessagesHandler(BaseTranslation):
             ]
         )
 
-        # Step 1: Extract all text content and images
+        # Step 1: Extract all text content, images, and tool calls
+        top_level_system_scanned: Final = (
+            ()
+            if hoisted_system_message is None or scan_only_tool_results
+            else self._extract_top_level_system_text(hoisted_system_message)
+        )
         extracted: Final = tuple(
             self._extract_input_text_and_images(
                 message=message,
@@ -501,17 +597,27 @@ class AnthropicMessagesHandler(BaseTranslation):
             )
             for msg_idx, message in enumerate(messages)
         )
-        scanned: Final = tuple(item for one_message in extracted for item in one_message.scanned)
+        scanned: Final = (
+            *top_level_system_scanned,
+            *(item for one_message in extracted for item in one_message.scanned),
+        )
         texts_to_check: Final = [item.text for item in scanned]  # mutable-ok: GenericGuardrailAPIInputs takes list[str]
         images_to_check: Final = [
             image for one_message in extracted for image in one_message.images
         ]  # mutable-ok: GenericGuardrailAPIInputs takes list[str]
+        scanned_tool_calls: Final = tuple(item for one_message in extracted for item in one_message.tool_calls)
+        tool_calls_to_check: Final = [
+            item.tool_call for item in scanned_tool_calls
+        ]  # mutable-ok: GenericGuardrailAPIInputs takes list[ChatCompletionToolCallChunk]
+        pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check)
 
-        # Step 2: Apply guardrail to all texts in batch
-        if texts_to_check:
+        # Step 2: Apply guardrail to all texts and tool calls in batch
+        if texts_to_check or tool_calls_to_check:
             inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
             if images_to_check:
                 inputs["images"] = images_to_check
+            if tool_calls_to_check:
+                inputs["tool_calls"] = tool_calls_to_check
             if tools_to_check:
                 inputs["tools"] = tools_to_check
             original_structured_messages: Final = structured_messages
@@ -570,9 +676,18 @@ class AnthropicMessagesHandler(BaseTranslation):
                     preserve_system_messages=has_midturn_system_message,
                 )
             else:
+                if guardrailed_texts and len(guardrailed_texts) != len(scanned):
+                    raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
+                self._apply_guardrail_tool_calls_to_input(
+                    messages=messages,
+                    scanned_tool_calls=scanned_tool_calls,
+                    pre_guardrail_tool_calls=pre_guardrail_tool_calls,
+                    returned_tool_calls=guardrailed_inputs.get("tool_calls"),
+                    guardrail_name=guardrail_to_apply.guardrail_name,
+                )
                 # Step 3: Map guardrail responses back to original message structure
                 await self._apply_guardrail_responses_to_input(
-                    messages=messages,
+                    data=data,
                     responses=guardrailed_texts,
                     scanned=scanned,
                 )
@@ -598,6 +713,19 @@ class AnthropicMessagesHandler(BaseTranslation):
         hoisted: Final = probe.get("messages") or []  # mutable-ok: API message payload
         return hoisted[0] if hoisted else None
 
+    @staticmethod
+    def _extract_top_level_system_text(hoisted_system_message: AllMessageValues) -> tuple[ScannedText, ...]:
+        content: Final = hoisted_system_message.get("content")
+        if isinstance(content, str):
+            return (ScannedText(content, SystemStringTarget()),)
+        if not isinstance(content, list):
+            return ()
+        return tuple(
+            ScannedText(text_str, SystemBlockTextTarget(block_idx))
+            for block_idx, block in enumerate(content)
+            if isinstance(block, dict) and isinstance(text_str := block.get("text"), str)
+        )
+
     @staticmethod
     def _openai_system_message_to_anthropic(
         message: Mapping[str, object],
@@ -852,9 +980,25 @@ class AnthropicMessagesHandler(BaseTranslation):
             for content_idx, content_item in enumerate(content)
             if isinstance(content_item, dict)
         )
+        tool_use_blocks: Final = (
+            ()
+            if scan_only_tool_results
+            else tuple(
+                (content_idx, content_item)
+                for content_idx, content_item in enumerate(content)
+                if isinstance(content_item, dict) and _is_client_tool_use(content_item)
+            )
+        )
         return ExtractedInput(
             scanned=tuple(item for block in blocks for item in block.scanned),
             images=tuple(image for block in blocks for image in block.images),
+            tool_calls=tuple(
+                ScannedToolCall(
+                    tool_call=AnthropicConfig.convert_tool_use_to_openai_format(content_item, tool_call_idx),
+                    target=ToolUseInputTarget(msg_idx, content_idx),
+                )
+                for tool_call_idx, (content_idx, content_item) in enumerate(tool_use_blocks)
+            ),
         )
 
     @classmethod
@@ -940,43 +1084,59 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     async def _apply_guardrail_responses_to_input(
         self,
-        messages: Sequence[_WritableMessage],
-        responses: list[str],
+        data: dict[str, object],  # mutable-ok: API message payload
+        responses: Sequence[str],
         scanned: tuple[ScannedText, ...],
     ) -> None:
         """
-        Apply guardrail responses back to input messages.
+        Apply guardrail responses back to the top-level system prompt and the input messages.
         """
+        raw_messages: Final = data.get("messages")
+        messages: Final[Sequence[_WritableMessage]] = raw_messages if isinstance(raw_messages, list) else ()
         for item, guardrail_response in zip(scanned, responses):
-            target = item.target
-            message = messages[target.msg_idx]
-            content = message.get("content", None)
-            if content is None:
-                continue
-
-            match target:
-                case MessageContentTarget():
-                    if isinstance(content, str):
-                        message["content"] = (
-                            guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
-                        )
-                case ContentBlockTextTarget(content_idx=content_idx):
-                    if isinstance(content, list):
-                        content[content_idx]["text"] = (
-                            guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
-                        )
-                case ToolResultStringTarget(content_idx=content_idx):
-                    if isinstance(content, list):
-                        content[content_idx]["content"] = (
-                            guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
-                        )
-                case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
-                    if isinstance(content, list):
-                        content[content_idx]["content"][block_idx]["text"] = (
+            match item.target:
+                case SystemStringTarget():
+                    if isinstance(data.get("system"), str):
+                        data["system"] = (
                             guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
                         )
+                case SystemBlockTextTarget(block_idx=block_idx):
+                    _write_back_system_block(data.get("system"), block_idx, guardrail_response)
+                case (
+                    MessageContentTarget()
+                    | ContentBlockTextTarget()
+                    | ToolResultStringTarget()
+                    | ToolResultBlockTextTarget() as message_target
+                ):
+                    _write_back_message_text(messages[message_target.msg_idx], message_target, guardrail_response)
                 case _:
-                    assert_never(target)
+                    assert_never(item.target)
+
+    @staticmethod
+    def _apply_guardrail_tool_calls_to_input(
+        messages: Sequence[_WritableMessage],
+        scanned_tool_calls: tuple[ScannedToolCall, ...],
+        pre_guardrail_tool_calls: tuple[_ToolCallShape, ...],
+        returned_tool_calls: Sequence[object] | None,
+        guardrail_name: str | None,
+    ) -> None:
+        post_guardrail_tool_calls: Final = _tool_call_shapes(
+            returned_tool_calls
+            if returned_tool_calls is not None and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
+            else tuple(item.tool_call for item in scanned_tool_calls)
+        )
+        rewritten: Final = tuple(
+            (item, after, _rewritten_tool_use_input(after.arguments))
+            for item, before, after in zip(scanned_tool_calls, pre_guardrail_tool_calls, post_guardrail_tool_calls)
+            if before != after
+        )
+        applicable: Final = tuple(
+            (item, after, rewritten_input) for item, after, rewritten_input in rewritten if rewritten_input is not None
+        )
+        if len(applicable) != len(rewritten):
+            raise unappliable_request_rewrite(guardrail_name)
+        for item, after, rewritten_input in applicable:
+            _write_back_tool_use(messages[item.target.msg_idx], item.target, after, rewritten_input)
 
     async def process_output_response(
         self,

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -19,6 +19,8 @@ from typing_extensions import assert_never
 
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
+from litellm.exceptions import MidStreamFallbackError
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.anthropic import (
     AppliedEdit,
     CompactionBlock,
@@ -58,6 +60,25 @@ def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]:
     return value if value else ()
 
 
+def _error_status_and_message(exc: Exception) -> tuple[int, str]:
+    if isinstance(exc, (BaseLLMException, MidStreamFallbackError)):
+        return exc.status_code, exc.message
+    return 500, str(exc) or "Upstream stream ended before completion"
+
+
+def _mid_stream_error_sse_event(exc: Exception) -> bytes:
+    from litellm.anthropic_interface.exceptions.exception_mapping_utils import (
+        AnthropicExceptionMapping,
+    )
+
+    status_code, message = _error_status_and_message(exc)
+    error_response = AnthropicExceptionMapping.transform_to_anthropic_error(
+        status_code=status_code,
+        raw_message=message,
+    )
+    return f"event: error\ndata: {json.dumps(error_response)}\n\n".encode()
+
+
 def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
     match delta_type:
         case "text_delta":
@@ -990,14 +1011,17 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         Async version of anthropic_sse_wrapper.
         Convert AnthropicStreamWrapper dict chunks to Server-Sent Events format.
         """
-        async for chunk in self:
-            if isinstance(chunk, dict):
-                event_type: str = str(chunk.get("type", "message"))
-                payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
-                yield payload.encode()
-            else:
-                # For non-dict chunks, forward the original value unchanged
-                yield chunk
+        try:
+            async for chunk in self:
+                if isinstance(chunk, dict):
+                    event_type: str = str(chunk.get("type", "message"))
+                    payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
+                    yield payload.encode()
+                else:
+                    yield chunk
+        except Exception as e:  # noqa: BLE001  # boundary before the socket: any upstream failure becomes an Anthropic error event
+            verbose_logger.exception("Anthropic Adapter - mid-stream error, emitting Anthropic error event: %s", e)
+            yield _mid_stream_error_sse_event(e)
 
     def _increment_content_block_index(self):
         self.current_content_block_index += 1

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -0,0 +1,388 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from itertools import accumulate
+from types import MappingProxyType
+from typing import Annotated, Final, Literal, Protocol, TypeAlias
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
+
+import litellm
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo, is_anthropic_oauth_key
+from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
+from litellm.llms.anthropic.experimental_pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION
+from litellm.types.router import LiteLLM_Params
+from litellm.types.utils import ModelResponse
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_HEADERS: Final = TypeAdapter(dict[str, str])
+_counter: Final = AnthropicCountTokensHandler()
+
+
+_NATIVE_HEADERS: Final = frozenset(
+    (
+        "host",
+        "accept",
+        "accept-encoding",
+        "connection",
+        "user-agent",
+        "content-length",
+        "content-type",
+        "x-api-key",
+        "anthropic-version",
+    )
+)
+
+_DEPLOYMENT_OPTIONS: Final = frozenset(
+    {
+        "model",
+        "api_key",
+        "api_base",
+        "custom_llm_provider",
+        "rpm",
+        "tpm",
+        "timeout",
+        "stream_timeout",
+        "max_retries",
+        "num_retries",
+        "max_parallel_requests",
+        "input_cost_per_token",
+        "output_cost_per_token",
+        "cache_read_input_token_cost",
+        "cache_creation_input_token_cost",
+        "cache_creation_input_token_cost_above_1hr",
+    }
+)
+
+
+class _StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+
+class _CacheControl(_StrictModel):
+    type: Literal["ephemeral"]
+    ttl: Literal["5m", "1h"] = "5m"
+
+
+class _Text(_StrictModel):
+    type: Literal["text"]
+    text: str = Field(min_length=1, pattern=r"\S")
+    cache_control: _CacheControl | None = None
+
+
+class _ToolUse(_StrictModel):
+    type: Literal["tool_use"]
+    id: str = Field(min_length=1)
+    name: str = Field(min_length=1)
+    input: Mapping[str, JsonValue]
+    cache_control: _CacheControl | None = None
+
+
+class _ResultText(_StrictModel):
+    type: Literal["text"]
+    text: str
+
+
+class _ToolResult(_StrictModel):
+    type: Literal["tool_result"]
+    tool_use_id: str = Field(min_length=1)
+    content: str | Annotated[tuple[_ResultText, ...], Field(strict=False)]
+    is_error: bool | None = None
+    cache_control: _CacheControl | None = None
+
+
+_Block: TypeAlias = Annotated[_Text | _ToolUse | _ToolResult, Field(discriminator="type")]
+
+
+class _Message(_StrictModel):
+    role: Literal["user", "assistant"]
+    content: str | Annotated[tuple[_Block, ...], Field(strict=False)]
+
+    def blocks(self) -> tuple[_Text | _ToolUse | _ToolResult, ...]:
+        return (_Text(type="text", text=self.content),) if isinstance(self.content, str) else tuple(self.content)
+
+
+class _Tool(_StrictModel):
+    name: str = Field(min_length=1)
+    description: str | None = None
+    input_schema: Mapping[str, JsonValue]
+    type: Literal["custom"] | None = None
+
+
+class _Request(_StrictModel):
+    messages: tuple[_Message, ...] = Field(min_length=1, strict=False)
+    system: str | Annotated[tuple[_ResultText, ...], Field(strict=False)] | None = None
+    tools: Annotated[tuple[_Tool, ...], Field(strict=False)] | None = None
+    model: str | None = None
+    max_tokens: int | None = None
+    stream: bool | None = None
+    temperature: float | int | None = None
+    top_p: float | int | None = None
+    top_k: int | None = None
+    stop_sequences: Annotated[tuple[str, ...], Field(strict=False)] | None = None
+    metadata: Mapping[str, JsonValue] | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class PromptPrefix:
+    prefix_body: Mapping[str, JsonValue]
+    fingerprint: str
+    fingerprints: tuple[str, ...]
+    ttl_seconds: int
+
+
+def _digest(value: object) -> str:
+    return hashlib.sha256(
+        json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
+    ).hexdigest()
+
+
+def _next_digest(previous: str, boundary: tuple[int, str, Mapping[str, JsonValue]]) -> str:
+    return _digest((previous, boundary))
+
+
+def parse_prompt(body: Mapping[str, JsonValue]) -> PromptPrefix | None:
+    try:
+        request: Final = _Request.model_validate(body)
+        blocks: Final = tuple(message.blocks() for message in request.messages)
+    except ValidationError:
+        return None
+    markers: Final = tuple(
+        (message_index, block_index, block.cache_control)
+        for message_index, message_blocks in enumerate(blocks)
+        for block_index, block in enumerate(message_blocks)
+        if block.cache_control is not None
+    )
+    if len(markers) != 1:
+        return None
+    message_end, block_end, marker = markers[0]
+    normalized: Final = _JSON_OBJECT.validate_python(request.model_dump(mode="json", exclude_none=True))
+    context: Final = MappingProxyType({key: normalized[key] for key in ("system", "tools") if key in normalized})
+    boundaries: Final = tuple(
+        (
+            message_index,
+            request.messages[message_index].role,
+            _JSON_OBJECT.validate_python(
+                block.model_dump(mode="json", exclude=MappingProxyType({"cache_control": True}), exclude_none=True)
+            ),
+        )
+        for message_index, message_blocks in enumerate(blocks[: message_end + 1])
+        for block_index, block in enumerate(message_blocks)
+        if message_index < message_end or block_index <= block_end
+    )
+    hashes: Final = tuple(
+        accumulate(boundaries, _next_digest, initial=_digest((_JSON_OBJECT.validate_python(context), marker.ttl)))
+    )[1:]
+    prefix_messages: Final = tuple(
+        _Message(
+            role=request.messages[message_index].role,
+            content=tuple(
+                block
+                for block_index, block in enumerate(message_blocks)
+                if message_index < message_end or block_index <= block_end
+            ),
+        )
+        for message_index, message_blocks in enumerate(blocks[: message_end + 1])
+    )
+    return PromptPrefix(
+        prefix_body=MappingProxyType(
+            _JSON_OBJECT.validate_python(
+                _Request(messages=prefix_messages, system=request.system, tools=request.tools).model_dump(
+                    mode="json", exclude_none=True
+                )
+            )
+        ),
+        fingerprint=hashes[-1],
+        fingerprints=tuple(reversed(hashes[-20:])),
+        ttl_seconds=3600 if marker.ttl == "1h" else 300,
+    )
+
+
+def cache_scope(
+    caller_key_hash: str,
+    deployment_id: str,
+    provider_key: str,
+    model: str,
+    anthropic_version: str = DEFAULT_ANTHROPIC_API_VERSION,
+) -> str:
+    return _digest((caller_key_hash, deployment_id, provider_key, model, anthropic_version))
+
+
+class _TTLUsage(BaseModel):
+    model_config = ConfigDict(strict=True)
+    ephemeral_5m_input_tokens: int = Field(default=0, ge=0)
+    ephemeral_1h_input_tokens: int = Field(default=0, ge=0)
+
+
+class _CacheUsage(BaseModel):
+    model_config = ConfigDict(strict=True)
+    cached_tokens: int = Field(default=0, ge=0)
+    cache_creation_tokens: int = Field(default=0, ge=0)
+    cache_creation_token_details: _TTLUsage | None = None
+
+
+class _Usage(BaseModel):
+    model_config = ConfigDict(strict=True)
+    prompt_tokens: int = Field(ge=0)
+    prompt_tokens_details: _CacheUsage
+
+
+class _Choice(BaseModel):
+    finish_reason: str = Field(min_length=1)
+
+
+class _Response(BaseModel):
+    model_config = ConfigDict(strict=True)
+    model: str
+    usage: _Usage
+    choices: tuple[_Choice, ...] = Field(min_length=1, strict=False)
+
+
+class _CountBody(BaseModel):
+    messages: Sequence[Mapping[str, JsonValue]]
+    tools: Sequence[Mapping[str, JsonValue]] | None = None
+    system: str | Sequence[Mapping[str, JsonValue]] | None = None
+
+
+class _CountResult(BaseModel):
+    input_tokens: Annotated[StrictInt, Field(ge=0)]
+
+
+class TokenCounter(Protocol):
+    async def __call__(self, model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None: ...
+
+
+def _count_objects(
+    values: Sequence[Mapping[str, JsonValue]],
+) -> list[dict[str, JsonValue]]:  # mutable-ok: the existing provider count API requires JSON lists/dicts
+    return [dict(value) for value in values]  # mutable-ok: serialize read-only inputs at the provider API boundary
+
+
+async def count_prompt_tokens(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
+    native: Final = _CountBody.model_validate(body)
+    try:
+        result: Final = _CountResult.model_validate(
+            await _counter.handle_count_tokens_request(
+                model=model,
+                messages=_count_objects(native.messages),
+                tools=_count_objects(native.tools) if native.tools is not None else None,
+                system=native.system,
+                api_key=api_key,
+                timeout=15.0,
+            )
+        )
+    except Exception:  # noqa: BLE001  # provider/count validation failures are unavailable estimates, not zero tokens
+        return None
+    return result.input_tokens
+
+
+@dataclass(frozen=True, slots=True)
+class NativePredictionTarget:
+    model: str
+    api_key: str
+
+
+@dataclass(frozen=True, slots=True)
+class UnsupportedPredictionTarget:
+    reason: Literal[
+        "unsupported_deployment_configuration",
+        "unsupported_provider_endpoint",
+        "unsupported_provider",
+        "unsupported_provider_credentials",
+    ]
+
+
+def resolve_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget | UnsupportedPredictionTarget:
+    configured_options: Final = frozenset(params.model_dump(exclude_defaults=True, exclude_none=True))
+    if configured_options - _DEPLOYMENT_OPTIONS:
+        return UnsupportedPredictionTarget("unsupported_deployment_configuration")
+    api_base: Final = AnthropicModelInfo.get_api_base(params.api_base)
+    if api_base not in ("https://api.anthropic.com", "https://api.anthropic.com/v1/messages"):
+        return UnsupportedPredictionTarget("unsupported_provider_endpoint")
+    try:
+        model, provider, _, _ = litellm.get_llm_provider(
+            model=params.model, custom_llm_provider=params.custom_llm_provider
+        )
+    except Exception:  # noqa: BLE001  # the shared provider resolver raises for unknown deployments
+        return UnsupportedPredictionTarget("unsupported_provider")
+    if provider != "anthropic":
+        return UnsupportedPredictionTarget("unsupported_provider")
+    api_key: Final = AnthropicModelInfo.get_api_key(params.api_key)
+    if api_key is None or not _supported_provider_key(api_key):
+        return UnsupportedPredictionTarget("unsupported_provider_credentials")
+    return NativePredictionTarget(model=model, api_key=api_key)
+
+
+def _supported_provider_key(api_key: str) -> bool:
+    return bool(api_key) and not is_anthropic_oauth_key(api_key)
+
+
+def supported_prediction_headers(headers: Mapping[str, str]) -> bool:
+    return all(
+        name.lower() != "anthropic-beta"
+        and (name.lower() != "anthropic-version" or value == DEFAULT_ANTHROPIC_API_VERSION)
+        for name, value in headers.items()
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class ObservedCachePrefix:
+    prefix: PromptPrefix
+    scope: str
+    cached_tokens: int
+    cache_creation_tokens: int
+
+
+def parse_observed_cache(
+    wire: httpx.Request, response_obj: ModelResponse, caller_key_hash: str, deployment_id: str
+) -> ObservedCachePrefix | None:
+    try:
+        response: Final = _Response.model_validate(response_obj, from_attributes=True)
+        body: Final = _JSON_OBJECT.validate_json(wire.content)
+        headers: Final = _HEADERS.validate_python(wire.headers)
+    except (ValidationError, RuntimeError, httpx.RequestNotRead):
+        return None
+    if (
+        wire.url.scheme != "https"
+        or wire.url.host != "api.anthropic.com"
+        or wire.url.path != "/v1/messages"
+        or wire.url.query
+        or wire.url.port not in (None, 443)
+    ):
+        return None
+    if (
+        frozenset(headers) - _NATIVE_HEADERS
+        or not supported_prediction_headers(headers)
+        or headers.get("anthropic-version") != DEFAULT_ANTHROPIC_API_VERSION
+    ):
+        return None
+    provider_key: Final = headers.get("x-api-key", "")
+    model: Final = body.get("model")
+    if not _supported_provider_key(provider_key) or not isinstance(model, str) or model != response.model:
+        return None
+    prefix: Final = parse_prompt(body)
+    if prefix is None:
+        return None
+    usage: Final = response.usage.prompt_tokens_details
+    cache_tokens: Final = usage.cached_tokens + usage.cache_creation_tokens
+    if cache_tokens <= 0 or cache_tokens > response.usage.prompt_tokens:
+        return None
+    split: Final = usage.cache_creation_token_details
+    if usage.cache_creation_tokens and split is None:
+        return None
+    if split is not None and (
+        split.ephemeral_5m_input_tokens + split.ephemeral_1h_input_tokens != usage.cache_creation_tokens
+        or (prefix.ttl_seconds == 300 and split.ephemeral_1h_input_tokens > 0)
+        or (prefix.ttl_seconds == 3600 and split.ephemeral_5m_input_tokens > 0)
+    ):
+        return None
+    return ObservedCachePrefix(
+        prefix=prefix,
+        scope=cache_scope(caller_key_hash, deployment_id, provider_key, model),
+        cached_tokens=cache_tokens,
+        cache_creation_tokens=usage.cache_creation_tokens,
+    )

litellm/llms/azure_ai/common_utils.py

@@ -144,10 +144,13 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
     def get_api_key(api_key: str | None = None) -> str | None:
         return api_key or litellm.api_key or get_secret_str("AZURE_AI_API_KEY")
 
+    @staticmethod
+    def get_api_version(api_version: str | None = None) -> str | None:
+        return api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
+
     @property
-    def api_version(self, api_version: str | None = None) -> str | None:
-        api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
-        return api_version
+    def api_version(self) -> str | None:
+        return AzureFoundryModelInfo.get_api_version()
 
     def get_token_counter(self) -> BaseTokenCounter | None:
         """

litellm/llms/base_llm/base_utils.py

@@ -50,6 +50,14 @@ class BaseLLMModelInfo(ABC):
         """
         return None
 
+    def get_model_cost_key(self, model: str) -> str | None:
+        """
+        Maps the model name a user sends to the key `litellm.model_cost` stores it under, when the two differ.
+        `get_model_info` tries this key once the exact `model` and `provider/model` keys miss. The default None means
+        the provider's user-facing names already match the cost map, so there is nothing extra to try.
+        """
+        return None
+
     @abstractmethod
     def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
         """

litellm/llms/base_llm/files/transformation.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Union
 
 import httpx
@@ -160,6 +160,15 @@ class BaseFilesConfig(BaseConfig):
     ) -> tuple[str, dict]:
         """Transform file list request into provider-specific format."""
 
+    def transform_list_files_next_request(
+        self,
+        raw_response: httpx.Response,
+        optional_params: Mapping[str, object],
+        litellm_params: dict,  # mutable-ok: carries provider stashes from the request transform to the response one
+    ) -> tuple[str, dict[str, str]] | None:
+        """Request for the page after `raw_response`, or None once the listing is complete."""
+        return None
+
     @abstractmethod
     def transform_list_files_response(
         self,
@@ -258,7 +267,7 @@ class BaseFileEndpoints(ABC):
         litellm_parent_otel_span: Span | None,
         llm_router: Router,
         **data: dict,
-    ) -> OpenAIFileObject:
+    ) -> FileDeleted:
         pass
 
     @abstractmethod

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import json
-from collections.abc import Callable, Iterator, Sequence
-from typing import Final, TypeVar
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from typing import Final, TypeVar, cast  # noqa: TID251  # a rebuilt chat row has no typed constructor across roles
 
 from pydantic import BaseModel
 
@@ -364,3 +364,67 @@ def merge_guardrailed_scoped_messages(
                 yield from appended
 
     return list(_merged())
+
+
+def _content_part_text(part: object) -> str | None:
+    if not isinstance(part, Mapping):
+        return None
+    text: Final = part.get("text")
+    return text if isinstance(text, str) else None
+
+
+def message_slot_texts(message: Mapping[str, object]) -> tuple[str, ...]:
+    content: Final = message.get("content")
+    if isinstance(content, str):
+        return (content,)
+    if isinstance(content, list):
+        return tuple(text for part in content if (text := _content_part_text(part)) is not None)
+    return ()
+
+
+def message_text_slot_count(message: AllMessageValues) -> int:
+    return len(message_slot_texts(message))
+
+
+def _part_with_text(part: object, text: str) -> object:
+    if not isinstance(part, Mapping):
+        return part
+    return {**part, "text": text}  # mutable-ok: content parts stay JSON-plain dicts
+
+
+def _content_with_slot_texts(content: Sequence[object], texts: Sequence[str]) -> Sequence[object]:
+    remaining_texts: Final = iter(texts)
+    return [  # mutable-ok: message content stays a JSON list
+        _part_with_text(part, next(remaining_texts)) if _content_part_text(part) is not None else part
+        for part in content
+    ]
+
+
+def message_with_slot_texts(message: AllMessageValues, texts: Sequence[str]) -> AllMessageValues | None:
+    """Swap one rewritten text into each text slot of a chat row, in order.
+
+    A slot is a string ``content`` or one list part carrying a string ``text``;
+    images and other parts ride along untouched. Returns None unless the counts
+    line up exactly, so a rewrite never lands on the wrong slot.
+    """
+    if message_text_slot_count(message) != len(texts):
+        return None
+    content: Final = message.get("content")
+    if not isinstance(content, (str, list)):
+        return message
+    rewritten_content: Final = texts[0] if isinstance(content, str) else _content_with_slot_texts(content, texts)
+    rewritten: Final = {**message, "content": rewritten_content}  # mutable-ok: chat rows stay JSON-plain dicts
+    return cast("AllMessageValues", rewritten)  # cast-ok: the same row with only its text slots swapped
+
+
+class UnappliableRequestRewrite(Exception):
+    def __init__(self, guardrail_name: str) -> None:
+        super().__init__(
+            f"Guardrail '{guardrail_name}' rewrote the request in a way this endpoint cannot apply, "
+            "so the request was rejected rather than sent unrewritten"
+        )
+        self.guardrail_name: Final = guardrail_name
+
+
+def unappliable_request_rewrite(guardrail_name: str | None) -> UnappliableRequestRewrite:
+    return UnappliableRequestRewrite(guardrail_name or "unknown")

litellm/llms/base_llm/realtime/transformation.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any
 
 import httpx
@@ -54,9 +55,12 @@ class BaseRealtimeConfig(ABC):
         message: str,
         model: str,
         session_configuration_request: str | None = None,
-    ) -> list[str]:
+    ) -> Sequence[str | bytes]:
         pass
 
+    async def pace_backend_send(self, message: bytes) -> None:
+        return None
+
     def is_setup_message(self, msg_obj: dict) -> bool:
         return False
 
@@ -79,7 +83,7 @@ class BaseRealtimeConfig(ABC):
         model: str,
         logging_session_id: str,
         session_configuration_request: str | None = None,
-    ) -> dict | OpenAIRealtimeStreamSessionEvents | None:
+    ) -> Mapping[str, object] | OpenAIRealtimeStreamSessionEvents | None:
         """
         Optional hook for providers that defer session setup until client `session.update`.
 

litellm/llms/bedrock/base_aws_llm.py

@@ -11,6 +11,7 @@ from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from functools import partial
 from threading import Lock
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, ParamSpec, TypeVar, cast, get_args, overload
 
 import httpx
@@ -96,6 +97,77 @@ def _assume_role_params(
             )
 
 
+_SecureTransportBool = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
+
+
+class _SecureTransportCondition(TypedDict):
+    Bool: ReadOnly[_SecureTransportBool]
+
+
+class _SessionPolicyStatement(TypedDict):
+    Sid: ReadOnly[str]
+    Effect: ReadOnly[Literal["Allow"]]
+    Action: ReadOnly[tuple[str, ...]]
+    Resource: ReadOnly[Literal["*"]]
+    Condition: ReadOnly[_SecureTransportCondition]
+
+
+class WebIdentitySessionPolicy(TypedDict):
+    Version: ReadOnly[Literal["2012-10-17"]]
+    Statement: ReadOnly[tuple[_SessionPolicyStatement, ...]]
+
+
+_WEB_IDENTITY_SESSION_POLICY_ACTIONS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+    {
+        "BedrockLiteLLM": (
+            "bedrock:InvokeModel",
+            "bedrock:InvokeModelWithResponseStream",
+            "bedrock:CountTokens",
+            "bedrock:Rerank",
+            "bedrock:Retrieve",
+            "bedrock:ListKnowledgeBases",
+            "bedrock:InvokeAgent",
+            "bedrock:ApplyGuardrail",
+            "bedrock:GetGuardrail",
+            "bedrock:ListGuardrails",
+        ),
+        "BedrockAgentCoreLiteLLM": (
+            "bedrock-agentcore:InvokeAgentRuntime",
+            "bedrock-agentcore:InvokeAgentRuntimeForUser",
+            "bedrock-agentcore:InvokeGateway",
+        ),
+        "ClaudePlatformLiteLLM": (
+            "aws-external-anthropic:CreateInference",
+            "aws-external-anthropic:CreateBatchInference",
+            "aws-external-anthropic:CancelBatchInference",
+            "aws-external-anthropic:DeleteBatchInference",
+            "aws-external-anthropic:CountTokens",
+            "aws-external-anthropic:Get*",
+            "aws-external-anthropic:List*",
+        ),
+        "BedrockMantleLiteLLM": ("bedrock-mantle:CreateInference",),
+    }
+)
+
+_SECURE_TRANSPORT_ONLY: Final = _SecureTransportCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
+
+
+def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
+    return WebIdentitySessionPolicy(
+        Version="2012-10-17",
+        Statement=tuple(
+            _SessionPolicyStatement(
+                Sid=sid,
+                Effect="Allow",
+                Action=actions,
+                Resource="*",
+                Condition=_SECURE_TRANSPORT_ONLY,
+            )
+            for sid, actions in _WEB_IDENTITY_SESSION_POLICY_ACTIONS.items()
+        ),
+    )
+
+
 class BedrockRequestTarget(BaseModel):
     aws_region_name: str
     aws_bedrock_runtime_endpoint: str | None
@@ -940,60 +1012,12 @@ class BaseAWSLLM(SignsRequestsWithAWS):
         # auth only (static creds + IRSA take other code paths).
         # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
-        bedrock_session_policy: Final = {
-            "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Sid": "BedrockLiteLLM",
-                    "Effect": "Allow",
-                    "Action": [
-                        "bedrock:InvokeModel",
-                        "bedrock:InvokeModelWithResponseStream",
-                        "bedrock:CountTokens",
-                        "bedrock:ApplyGuardrail",
-                        "bedrock:GetGuardrail",
-                        "bedrock:ListGuardrails",
-                    ],
-                    "Resource": "*",
-                    "Condition": {"Bool": {"aws:SecureTransport": "true"}},
-                },
-                # Claude Platform on AWS (added by #27678 for the
-                # ``bedrock/claude_platform/<model>`` route) lives under
-                # a separate IAM action namespace; without these entries
-                # the OIDC path 403s on every claude_platform request
-                # even with a fully permissive identity policy (#30200).
-                {
-                    "Sid": "ClaudePlatformLiteLLM",
-                    "Effect": "Allow",
-                    "Action": [
-                        "aws-external-anthropic:CreateInference",
-                        "aws-external-anthropic:CreateBatchInference",
-                        "aws-external-anthropic:CancelBatchInference",
-                        "aws-external-anthropic:DeleteBatchInference",
-                        "aws-external-anthropic:CountTokens",
-                        "aws-external-anthropic:Get*",
-                        "aws-external-anthropic:List*",
-                    ],
-                    "Resource": "*",
-                    "Condition": {"Bool": {"aws:SecureTransport": "true"}},
-                },
-                {
-                    "Sid": "BedrockMantleLiteLLM",
-                    "Effect": "Allow",
-                    "Action": [
-                        "bedrock-mantle:CreateInference",
-                    ],
-                    "Resource": "*",
-                    "Condition": {"Bool": {"aws:SecureTransport": "true"}},
-                },
-            ],
-        }
         assume_role_params: Final = {
             "RoleArn": aws_role_name,
             "RoleSessionName": aws_session_name,
             "WebIdentityToken": oidc_token,
             "DurationSeconds": 3600,
-            "Policy": json.dumps(bedrock_session_policy, separators=(",", ":")),
+            "Policy": json.dumps(build_web_identity_session_policy(), separators=(",", ":")),
         }
 
         # Add ExternalId parameter if provided

litellm/llms/bedrock/files/transformation.py

@@ -1,14 +1,18 @@
 import base64
 import json
 import os
+import posixpath
 import time
+import xml.etree.ElementTree as ET
 from collections.abc import Iterable, Mapping, MutableMapping, Sequence
 from contextlib import suppress
+from dataclasses import dataclass
+from datetime import datetime
 from functools import cache
 from itertools import chain
 from types import MappingProxyType
 from typing import Any, Final, Literal, TypeAlias, TypedDict
-from urllib.parse import unquote
+from urllib.parse import quote, unquote, urlencode
 
 import httpx
 from httpx import Headers, Response
@@ -23,6 +27,7 @@ from litellm.files.utils import FilesAPIUtils
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.cloud_storage_security import (
     BEDROCK_MANAGED_S3_BATCH_PREFIX,
+    BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
     BEDROCK_MANAGED_S3_PREFIXES,
     BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
     build_managed_cloud_object_name,
@@ -62,6 +67,10 @@ from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resol
 
 S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers"
 
+LIST_FILES_PURPOSE_PARAM: Final = "_s3_list_files_purpose"
+
+LIST_FILES_LOCATION_PARAM: Final = "_s3_list_files_location"
+
 
 class _S3DeleteContext(BaseModel):
     file_id: str = Field(min_length=1)
@@ -152,6 +161,13 @@ class _BedrockS3RequestParams(BaseModel):
     s3_endpoint_url: str | None = None
 
 
+@dataclass(frozen=True, slots=True)
+class _S3RequestTarget:
+    endpoint_url: str
+    aws_region_name: str
+    request_params: _BedrockS3RequestParams
+
+
 class _TrustedS3ModelCredentials(BaseModel):
     """The S3 buckets the server trusts file ids against, from the deployment snapshot."""
 
@@ -248,6 +264,128 @@ def _validate_file_id_against_configured_buckets(
     return validate_against(configured_bucket_names[-1])
 
 
+_REJECTED_FILE_ID_REQUEST_URL: Final = "https://litellm.ai"
+
+
+def _rejected_file_id(reason: ValueError) -> BedrockError:
+    message: Final = str(reason)
+    return BedrockError(
+        status_code=400,
+        message=message,
+        response=httpx.Response(
+            status_code=400,
+            text=message,
+            request=httpx.Request(method="GET", url=_REJECTED_FILE_ID_REQUEST_URL),
+        ),
+    )
+
+
+def _resolve_managed_s3_object(file_id: str, litellm_params: Mapping[str, object]) -> tuple[str, str]:
+    configured_bucket_names: Final = get_configured_s3_bucket_names(litellm_params)
+    allow_legacy_cloud_file_ids: Final = should_allow_legacy_cloud_file_ids(litellm_params)
+    try:
+        return _validate_file_id_against_configured_buckets(
+            s3_uri=extract_s3_uri_from_file_id(file_id),
+            configured_bucket_names=configured_bucket_names,
+            allow_legacy_cloud_file_ids=allow_legacy_cloud_file_ids,
+        )
+    except ValueError as reason:
+        raise _rejected_file_id(reason) from reason
+
+
+_ANY_MANAGED_LISTING_PREFIX: Final = os.path.commonprefix(BEDROCK_MANAGED_S3_PREFIXES)
+_MANAGED_LISTING_PREFIX_BY_PURPOSE: Final = MappingProxyType(
+    {
+        "batch": os.path.commonprefix((BEDROCK_MANAGED_S3_BATCH_PREFIX, BEDROCK_MANAGED_S3_UPLOAD_PREFIX)),
+        "batch_output": BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
+    }
+)
+
+
+_EMPTY_LISTING_QUERY: Final = (("list-type", "2"), ("max-keys", "0"))
+
+
+def _managed_listing_prefix(configured_prefix: str, purpose: str | None) -> str | None:
+    managed_prefix: Final = _MANAGED_LISTING_PREFIX_BY_PURPOSE.get(purpose) if purpose else _ANY_MANAGED_LISTING_PREFIX
+    if managed_prefix is None:
+        return None
+    return f"{configured_prefix}/{managed_prefix}" if configured_prefix else managed_prefix
+
+
+def _listing_query(configured_prefix: str, purpose: str | None) -> tuple[tuple[str, str], ...]:
+    listing_prefix: Final = _managed_listing_prefix(configured_prefix, purpose)
+    if listing_prefix is None:
+        return _EMPTY_LISTING_QUERY
+    return (("list-type", "2"), ("prefix", listing_prefix))
+
+
+def _requested_listing_purpose(litellm_params: Mapping[str, object]) -> str | None:
+    requested_purpose: Final = litellm_params.get(LIST_FILES_PURPOSE_PARAM)
+    return requested_purpose if isinstance(requested_purpose, str) else None
+
+
+def _walked_listing_purpose(litellm_params: Mapping[str, object]) -> str | None:
+    walked_purpose: Final = litellm_params.get(LIST_FILES_LOCATION_PARAM)
+    return walked_purpose if isinstance(walked_purpose, str) else _requested_listing_purpose(litellm_params)
+
+
+def _output_location_still_unlisted(litellm_params: Mapping[str, object]) -> bool:
+    if _walked_listing_purpose(litellm_params) is not None:
+        return False
+    return _listing_bucket_name(litellm_params, "batch_output") != _listing_bucket_name(litellm_params, None)
+
+
+def _listing_bucket_name(litellm_params: Mapping[str, object], purpose: str | None) -> str:
+    if purpose != "batch_output":
+        return get_configured_s3_bucket_name(litellm_params)
+    trusted: Final = _trusted_s3_model_credentials(litellm_params)
+    return (
+        trusted.s3_output_bucket_name
+        or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
+        or get_configured_s3_bucket_name(litellm_params)
+    )
+
+
+def _listed_object_created_at(entry: ET.Element) -> int:
+    last_modified: Final = entry.findtext("{*}LastModified")
+    if not last_modified:
+        return 0
+    return int(datetime.fromisoformat(last_modified.replace("Z", "+00:00")).timestamp())
+
+
+def _listed_managed_file(
+    entry: ET.Element,
+    bucket_name: str,
+    configured_bucket_name: str,
+    allow_legacy_cloud_file_ids: bool,
+) -> OpenAIFileObject | None:
+    object_key: Final = entry.findtext("{*}Key")
+    if not object_key:
+        return None
+    file_id: Final = f"s3://{bucket_name}/{object_key}"
+    try:
+        validate_managed_cloud_file_id(
+            file_id=file_id,
+            scheme="s3://",
+            configured_bucket_name=configured_bucket_name,
+            allowed_object_prefixes=BEDROCK_MANAGED_S3_PREFIXES,
+            allow_legacy_cloud_file_ids=allow_legacy_cloud_file_ids,
+        )
+    except ValueError:
+        return None
+    _, configured_prefix = split_configured_cloud_bucket_name(configured_bucket_name)
+    relative_key: Final = object_key[len(configured_prefix) + 1 :] if configured_prefix else object_key
+    return OpenAIFileObject(
+        id=file_id,
+        bytes=int(entry.findtext("{*}Size") or 0),
+        created_at=_listed_object_created_at(entry),
+        filename=posixpath.basename(object_key),
+        object="file",
+        purpose="batch_output" if relative_key.startswith(BEDROCK_MANAGED_S3_OUTPUT_PREFIX) else "batch",
+        status="uploaded",
+    )
+
+
 def _uploaded_object_size(litellm_params: Mapping[str, object], raw_response: Response) -> int:
     """
     S3 answers PutObject with an empty body, so the stored object size comes from the
@@ -1213,18 +1351,86 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
     def transform_list_files_request(
         self,
         purpose: str | None,
-        optional_params: dict,
-        litellm_params: dict,
-    ) -> tuple[str, dict]:
-        raise NotImplementedError("BedrockFilesConfig does not support file listing")
+        optional_params: Mapping[str, object],
+        litellm_params: MutableMapping[str, object],
+    ) -> tuple[str, dict[str, str]]:
+        litellm_params[LIST_FILES_PURPOSE_PARAM] = purpose  # rebind-ok: handed to the response transform
+        litellm_params[LIST_FILES_LOCATION_PARAM] = purpose  # rebind-ok: names the location the next page walks
+        return self._signed_listing_request(purpose, optional_params, litellm_params, continuation_token=None)
+
+    def transform_list_files_next_request(
+        self,
+        raw_response: httpx.Response,
+        optional_params: Mapping[str, object],
+        litellm_params: MutableMapping[str, object],
+    ) -> tuple[str, dict[str, str]] | None:
+        if raw_response.status_code >= 400:
+            return None
+        continuation_token: Final = ET.fromstring(raw_response.content).findtext("{*}NextContinuationToken")
+        if continuation_token:
+            return self._signed_listing_request(
+                _walked_listing_purpose(litellm_params), optional_params, litellm_params, continuation_token
+            )
+        if not _output_location_still_unlisted(litellm_params):
+            return None
+        litellm_params[LIST_FILES_LOCATION_PARAM] = "batch_output"  # rebind-ok: the input location is fully listed
+        return self._signed_listing_request("batch_output", optional_params, litellm_params, continuation_token=None)
+
+    def _signed_listing_request(
+        self,
+        purpose: str | None,
+        optional_params: Mapping[str, object],
+        litellm_params: MutableMapping[str, object],
+        continuation_token: str | None,
+    ) -> tuple[str, dict[str, str]]:
+        bucket_name, configured_prefix = split_configured_cloud_bucket_name(
+            _listing_bucket_name(litellm_params, purpose)
+        )
+        target: Final = self._s3_request_target(optional_params=optional_params, litellm_params=litellm_params)
+        url: Final = f"{target.endpoint_url}/{bucket_name}/"
+        listing_query: Final = _listing_query(configured_prefix, purpose)
+        continuation_query: Final = (("continuation-token", continuation_token),) if continuation_token else ()
+        query: Final[dict[str, str]] = dict(  # mutable-ok: the base files contract returns the query as a dict
+            listing_query + continuation_query
+        )
+        signed_headers: Final = self._sign_s3_request_without_body(
+            method="GET",
+            api_base=f"{url}?{urlencode(query, quote_via=quote, safe='')}",
+            aws_region_name=target.aws_region_name,
+            request_params=target.request_params,
+        )
+        litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM] = signed_headers  # rebind-ok: handed to validate_environment
+        return url, query
 
     def transform_list_files_response(
         self,
         raw_response: httpx.Response,
         logging_obj: LiteLLMLoggingObj,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
     ) -> list[OpenAIFileObject]:
-        raise NotImplementedError("BedrockFilesConfig does not support file listing")
+        if raw_response.status_code >= 400:
+            raise BedrockError(
+                status_code=raw_response.status_code,
+                message=raw_response.text,
+                headers=raw_response.headers,
+                response=raw_response,
+            )
+        purpose: Final = _requested_listing_purpose(litellm_params)
+        configured_bucket_name: Final = _listing_bucket_name(litellm_params, _walked_listing_purpose(litellm_params))
+        allow_legacy_cloud_file_ids: Final = should_allow_legacy_cloud_file_ids(litellm_params)
+        listing: Final = ET.fromstring(raw_response.content)
+        bucket_name: Final = (
+            listing.findtext("{*}Name") or split_configured_cloud_bucket_name(configured_bucket_name)[0]
+        )
+        listed_files: Final = (
+            _listed_managed_file(entry, bucket_name, configured_bucket_name, allow_legacy_cloud_file_ids)
+            for entry in listing.iterfind("{*}Contents")
+        )
+        return [  # mutable-ok: the base files contract returns a list
+            listed_file
+            for listed_file in listed_files
+            if listed_file is not None and (purpose is None or listed_file.purpose == purpose)
+        ]
 
     def transform_file_content_request(
         self,
@@ -1255,39 +1461,54 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         optional_params: Mapping[str, object],
         litellm_params: MutableMapping[str, object],
     ) -> tuple[str, dict[str, str]]:
-        s3_uri: Final = extract_s3_uri_from_file_id(file_id)
-        bucket_name, object_key = _validate_file_id_against_configured_buckets(
-            s3_uri=s3_uri,
-            configured_bucket_names=get_configured_s3_bucket_names(litellm_params),
-            allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(litellm_params),
+        bucket_name, object_key = _resolve_managed_s3_object(file_id=file_id, litellm_params=litellm_params)
+        target: Final = self._s3_request_target(optional_params=optional_params, litellm_params=litellm_params)
+        url: Final = f"{target.endpoint_url}/{bucket_name}/{encode_s3_object_key_for_url(object_key)}"
+        signed_headers: Final = self._sign_s3_request_without_body(
+            method=method,
+            api_base=url,
+            aws_region_name=target.aws_region_name,
+            request_params=target.request_params,
         )
+        litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM] = signed_headers  # rebind-ok: handed to validate_environment
+        return url, {}  # mutable-ok: the base files contract returns the query as a dict
 
-        request_params: Final = _BedrockS3RequestParams.model_validate({**litellm_params, **optional_params})
-
+    def _s3_request_target(
+        self,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> _S3RequestTarget:
+        """
+        The shared files handler passes optional_params={}, so AWS credentials and
+        region arrive via litellm_params here (unlike the upload path).
+        s3_region_name wins over aws_region_name, same priority as get_complete_file_url.
+        """
+        request_params: Final = _BedrockS3RequestParams.model_validate(
+            MappingProxyType({**litellm_params, **optional_params})
+        )
         region_preference: Final = request_params.s3_region_name or request_params.aws_region_name
-        region_params: Final[dict[str, str | None]] = {"aws_region_name": region_preference}
-        aws_region_name: Final = self._get_aws_region_name(optional_params=region_params, model="")
-
-        s3_endpoint_url: Final = (
+        aws_region_name: Final = self._get_aws_region_name(
+            optional_params={"aws_region_name": region_preference},  # mutable-ok: BaseAWSLLM takes a dict
+            model="",
+        )
+        endpoint_url: Final = (
             request_params.s3_endpoint_url or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
         ).rstrip("/")
-        url: Final = f"{s3_endpoint_url}/{bucket_name}/{encode_s3_object_key_for_url(object_key)}"
-
-        litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM] = self._sign_s3_request_without_body(
-            api_base=url,
-            aws_region_name=aws_region_name,
-            request_params=request_params,
-            method=method,
+        return _S3RequestTarget(
+            endpoint_url=endpoint_url, aws_region_name=aws_region_name, request_params=request_params
         )
-        return url, {}
 
     def _sign_s3_request_without_body(
         self,
+        method: Literal["GET", "DELETE"],
         api_base: str,
         aws_region_name: str,
         request_params: _BedrockS3RequestParams,
-        method: Literal["GET", "DELETE"] = "GET",
-    ) -> dict[str, str]:
+    ) -> Mapping[str, str]:
+        """
+        SigV4-sign a bodiless S3 request (GetObject, DeleteObject, ListObjectsV2),
+        mirroring `_sign_s3_request` (PUT).
+        """
         try:
             import hashlib
 
@@ -1313,11 +1534,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         aws_request: Final = AWSRequest(  # any-ok: botocore AWSRequest is untyped
             method=method,
             url=api_base,
-            headers={"x-amz-content-sha256": empty_body_hash},
+            headers={"x-amz-content-sha256": empty_body_hash},  # mutable-ok: botocore AWSRequest takes a dict
         )
         auth: Final = S3SigV4Auth(credentials, "s3", aws_region_name)  # any-ok: botocore untyped
         auth.add_auth(aws_request)  # any-ok: botocore request mutation is untyped
-        return dict(aws_request.headers)  # any-ok: botocore headers are untyped
+        return MappingProxyType(dict(aws_request.headers))  # any-ok: botocore headers are untyped
 
     def transform_file_content_response(
         self,
@@ -1330,6 +1551,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
                 status_code=raw_response.status_code,
                 message=raw_response.text,
                 headers=raw_response.headers,
+                response=raw_response,
             )
         return HttpxBinaryResponseContent(response=raw_response)
 

litellm/llms/bedrock/realtime/handler.py

@@ -7,13 +7,21 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
 import asyncio
 import contextlib
 import json
-from collections.abc import AsyncIterator, Mapping
-from typing import Final, Protocol
+from collections.abc import AsyncIterator, Mapping, MutableMapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final, NoReturn, Protocol
 
 from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import _redact_string, verbose_proxy_logger
+from litellm.constants import (
+    BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
+    BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY,
+    BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
+    REALTIME_SESSION_SUCCESS_LOGGED_KEY,
+)
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
@@ -28,6 +36,32 @@ from .transformation import BedrockRealtimeConfig
 _CLIENT_MODALITIES_ADAPTER: Final[TypeAdapter["list[str] | None"]] = TypeAdapter(list[str] | None)
 _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
+_EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
+
+_BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType(
+    {
+        "AccessDeniedException": 403,
+        "ConflictException": 400,
+        "InternalServerException": 500,
+        "ModelErrorException": 424,
+        "ModelNotReadyException": 429,
+        "ModelStreamErrorException": 424,
+        "ModelTimeoutException": 408,
+        "ResourceNotFoundException": 404,
+        "ServiceQuotaExceededException": 400,
+        "ServiceUnavailableException": 503,
+        "ThrottlingException": 429,
+        "ValidationException": 400,
+    }
+)
+
+
+def _as_bedrock_error(error: BaseException) -> BaseException:
+    status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__)
+    if status_code is None:
+        return error
+    return BedrockError(status_code=status_code, message=f"{type(error).__name__}: {error}")
+
 
 def _json_dict(value: JsonValue) -> dict[str, JsonValue]:
     return value if isinstance(value, dict) else {}
@@ -51,6 +85,8 @@ def _should_log_event(openai_message: Mapping[str, object]) -> bool:
 class RealtimeClientWebSocket(Protocol):
     """The client-facing websocket surface the realtime bridge talks to."""
 
+    scope: MutableMapping[str, object]  # mutable-ok: the ASGI scope is the per-connection state store
+
     async def receive_text(self) -> str: ...
 
     async def send_text(self, data: str) -> None: ...
@@ -85,6 +121,81 @@ class BedrockBidirectionalStream(Protocol):
     async def await_output(self) -> tuple[object, BedrockOutputStream]: ...
 
 
+@dataclass(frozen=True, slots=True)
+class _BridgeOutcome:
+    logged_events: tuple[OpenAIRealtimeEvents, ...]
+    provider_failure: BaseException | None
+    client_disconnected: bool
+
+
+async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]:
+    if initial_message is not None:
+        yield initial_message
+    while True:
+        try:
+            yield await client_ws.receive_text()
+        except Exception as e:  # noqa: BLE001  # any receive failure means the client is gone
+            verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
+            return
+
+
+def _pending_session_update(scope: Mapping[str, object]) -> str | None:
+    """A fallback attempt on the same websocket replays the session.update the failed attempt never acked."""
+    if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
+        committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
+        raise BedrockError(
+            status_code=400,
+            message=(
+                "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+                + (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
+            ),
+        )
+    pending: Final = scope.get(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY)
+    return pending if isinstance(pending, str) else None
+
+
+def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
+    error: Final = _as_bedrock_error(failure)
+    verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", _redact_string(str(error)))
+    if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
+        scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = _redact_string(str(error))
+    raise error from failure
+
+
+def _parse_client_message(message: str) -> Mapping[str, JsonValue]:
+    try:
+        return _json_dict(_CLIENT_MESSAGE_ADAPTER.validate_json(message))
+    except ValueError:
+        return _EMPTY_JSON_OBJECT
+
+
+async def _ack_session_update(
+    client_ws: RealtimeClientWebSocket,
+    bedrock_stream: BedrockBidirectionalStream,
+    transformation_config: BedrockRealtimeConfig,
+    model: str,
+    logging_obj: LiteLLMLogging | None,
+    parsed_client_message: Mapping[str, JsonValue],
+) -> bool:
+    """Ack the client's session.update once Bedrock accepted the stream; False means the client is gone."""
+    await bedrock_stream.await_output()
+    client_ws.scope.pop(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY, None)
+    client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True  # rebind-ok: scope outlives the attempt
+    if logging_obj is None:
+        return True
+    requested_modalities: Final = _CLIENT_MODALITIES_ADAPTER.validate_python(
+        _json_dict(parsed_client_message.get("session")).get("modalities")
+    )
+    try:
+        await client_ws.send_text(
+            json.dumps(transformation_config.session_updated_event(model, logging_obj, requested_modalities))
+        )
+    except Exception as e:  # noqa: BLE001  # any send failure means the client is gone
+        verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
+        return False
+    return True
+
+
 class BedrockRealtime(BaseAWSLLM):
     """Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
 
@@ -132,6 +243,8 @@ class BedrockRealtime(BaseAWSLLM):
         except ImportError:
             raise ImportError("Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime")
 
+        pending_session_update: Final = _pending_session_update(websocket.scope)
+
         # Get AWS region
         if aws_region_name is None:
             optional_params: Final = {
@@ -190,90 +303,106 @@ class BedrockRealtime(BaseAWSLLM):
 
         transformation_config: Final = BedrockRealtimeConfig()
 
-        try:
-            # Initialize the bidirectional stream
-            bedrock_stream: Final = await open_bidirectional_stream()
+        bedrock_stream: Final = await open_bidirectional_stream()
 
-            verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
+        verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
 
+        if pending_session_update is None:
             await websocket.send_text(json.dumps(transformation_config.session_created_event(model, logging_obj)))
             verbose_proxy_logger.debug("Bedrock Realtime: sent session.created to client on connect")
 
-            # Track state for transformation
-            session_state: Final[RealtimeResponseTransformInput] = {
-                "current_output_item_id": None,
-                "current_response_id": None,
-                "current_conversation_id": None,
-                "current_delta_chunks": None,
-                "current_item_chunks": None,
-                "current_delta_type": None,
-                "session_configuration_request": None,
-            }
+        # Track state for transformation
+        session_state: Final[RealtimeResponseTransformInput] = {
+            "current_output_item_id": None,
+            "current_response_id": None,
+            "current_conversation_id": None,
+            "current_delta_chunks": None,
+            "current_item_chunks": None,
+            "current_delta_type": None,
+            "session_configuration_request": None,
+        }
 
-            # Create tasks for bidirectional forwarding
-            client_to_bedrock_task: Final = asyncio.create_task(
-                self._forward_client_to_bedrock(
-                    websocket,
-                    bedrock_stream,
-                    transformation_config,
-                    model,
-                    session_state,
-                    logging_obj,
+        outcome: Final = await self._bridge(
+            websocket,
+            bedrock_stream,
+            transformation_config,
+            model,
+            session_state,
+            logging_obj,
+            initial_message=pending_session_update,
+        )
+
+        logged_events: Final = (
+            *outcome.logged_events,
+            *(
+                leftover_event
+                for leftover_event in transformation_config.leftover_usage_done_events()
+                if _should_log_event(leftover_event)
+            ),
+        )
+        if logged_events:
+            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+                logging_obj.dispatch_success_handlers(
+                    list(logged_events),  # mutable-ok: realtime spend logging requires a list result
+                    prefer_async_handlers=True,
                 )
             )
+            logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
 
-            async def forward_bedrock_and_collect_logged_events() -> tuple[OpenAIRealtimeEvents, ...]:
-                return tuple(
-                    [
-                        event
-                        async for event in self._forward_bedrock_to_client(
-                            bedrock_stream,
-                            websocket,
-                            transformation_config,
-                            model,
-                            logging_obj,
-                            session_state,
-                        )
-                    ]
-                )
-
-            bedrock_to_client_task: Final = asyncio.create_task(forward_bedrock_and_collect_logged_events())
-
-            # Wait for both tasks to complete
-            await asyncio.gather(
-                client_to_bedrock_task,
-                bedrock_to_client_task,
-                return_exceptions=True,
+        if outcome.provider_failure is None:
+            return
+        if outcome.client_disconnected:
+            verbose_proxy_logger.debug(
+                "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
             )
+            return
+        _raise_provider_failure(websocket.scope, outcome.provider_failure)
 
-            forwarded_logged_events: Final = (
-                bedrock_to_client_task.result()
-                if not bedrock_to_client_task.cancelled() and bedrock_to_client_task.exception() is None
-                else ()
-            )
-            logged_events: Final = (
-                *forwarded_logged_events,
-                *(
-                    leftover_event
-                    for leftover_event in transformation_config.leftover_usage_done_events()
-                    if _should_log_event(leftover_event)
-                ),
-            )
-            if logged_events:
-                GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
-                    logging_obj.dispatch_success_handlers(
-                        list(logged_events),  # mutable-ok: realtime spend logging requires a list result
-                        prefer_async_handlers=True,
-                    )
-                )
+    async def _bridge(
+        self,
+        websocket: RealtimeClientWebSocket,
+        bedrock_stream: BedrockBidirectionalStream,
+        transformation_config: BedrockRealtimeConfig,
+        model: str,
+        session_state: RealtimeResponseTransformInput,
+        logging_obj: LiteLLMLogging,
+        initial_message: str | None,
+    ) -> _BridgeOutcome:
+        """Run both forwarding directions until the client leaves or either side fails."""
+        logged: Final[list[OpenAIRealtimeEvents]] = []  # mutable-ok: events forwarded before a failure are still spend
 
-        except Exception as e:
-            verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e)
-            try:
-                await websocket.close(code=1011, reason=_redact_string(f"Internal error: {e}"))
-            except Exception:
-                pass
-            raise
+        async def collect_logged_events() -> None:
+            async for event in self._forward_bedrock_to_client(
+                bedrock_stream, websocket, transformation_config, model, logging_obj, session_state
+            ):
+                logged.append(event)
+
+        client_task: Final = asyncio.create_task(
+            self._forward_client_to_bedrock(
+                websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message
+            )
+        )
+        bedrock_task: Final = asyncio.create_task(collect_logged_events())
+
+        await asyncio.wait((client_task, bedrock_task), return_when=asyncio.FIRST_COMPLETED)
+        client_disconnected: Final = (
+            client_task.done() and not client_task.cancelled() and client_task.exception() is None
+        )
+        client_task.cancel()
+        bedrock_task.cancel()
+        client_outcome, bedrock_outcome = await asyncio.gather(client_task, bedrock_task, return_exceptions=True)
+
+        return _BridgeOutcome(
+            logged_events=tuple(logged),
+            provider_failure=(
+                client_outcome
+                if isinstance(client_outcome, Exception)
+                else bedrock_outcome
+                if isinstance(bedrock_outcome, Exception)
+                else None
+            ),
+            client_disconnected=client_disconnected,
+        )
 
     async def _forward_client_to_bedrock(
         self,
@@ -283,8 +412,12 @@ class BedrockRealtime(BaseAWSLLM):
         model: str,
         session_state: RealtimeResponseTransformInput,
         logging_obj: LiteLLMLogging | None = None,
-    ):
-        """Forward messages from client WebSocket to Bedrock stream."""
+        initial_message: str | None = None,
+    ) -> None:
+        """Forward messages from client WebSocket to Bedrock stream.
+
+        Returns once the client is gone; provider failures (input stream or readiness) propagate to the caller.
+        """
         from aws_sdk_bedrock_runtime.models import (
             BidirectionalInputPayloadPart,
             InvokeModelWithBidirectionalStreamInputChunk,
@@ -299,41 +432,28 @@ class BedrockRealtime(BaseAWSLLM):
             verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])
 
         try:
-            while True:
-                # Receive message from client
-                message = await client_ws.receive_text()
+            async for message in _client_messages(client_ws, initial_message):
                 verbose_proxy_logger.debug("Bedrock Realtime: Received from client: %s", message[:200])
+                parsed_client_message = _parse_client_message(message)
+                is_session_update = _json_str(parsed_client_message.get("type")) == "session.update"
+                if is_session_update:
+                    client_ws.scope[BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY] = (
+                        message  # rebind-ok: scope outlives the attempt
+                    )
 
-                # Transform OpenAI format to Bedrock format
                 transformed_messages = transformation_config.transform_realtime_request(
                     message=message,
                     model=model,
                     session_configuration_request=session_state.get("session_configuration_request"),
                 )
-
-                # Send transformed messages to Bedrock
                 for bedrock_message in transformed_messages:
                     await send_to_bedrock(bedrock_message)
 
-                if logging_obj is not None:
-                    client_message_type: str | None = None
-                    requested_modalities: list[str] | None = None
-                    with contextlib.suppress(Exception):
-                        parsed_client_message = _json_dict(_CLIENT_MESSAGE_ADAPTER.validate_json(message))
-                        client_message_type = _json_str(parsed_client_message.get("type"))
-                        if client_message_type == "session.update":
-                            requested_modalities = _CLIENT_MODALITIES_ADAPTER.validate_python(
-                                _json_dict(parsed_client_message.get("session")).get("modalities")
-                            )
-                    if client_message_type == "session.update":
-                        await client_ws.send_text(
-                            json.dumps(
-                                transformation_config.session_updated_event(model, logging_obj, requested_modalities)
-                            )
-                        )
-
-        except Exception as e:
-            verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
+                if is_session_update and not await _ack_session_update(
+                    client_ws, bedrock_stream, transformation_config, model, logging_obj, parsed_client_message
+                ):
+                    break
+        finally:
             for close_message in transformation_config.session_close_messages():
                 with contextlib.suppress(Exception):
                     await send_to_bedrock(close_message)
@@ -349,68 +469,71 @@ class BedrockRealtime(BaseAWSLLM):
         logging_obj: LiteLLMLogging,
         session_state: RealtimeResponseTransformInput,
     ) -> AsyncIterator[OpenAIRealtimeEvents]:
-        """Forward messages from Bedrock to the client, yielding the ones to record for spend logging."""
-        try:
-            while True:
-                # Receive from Bedrock
-                output = await bedrock_stream.await_output()
-                result = await output[1].receive()
+        """Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
 
-                if result is None:
-                    verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
-                    break
+        Provider failures propagate to the caller; the client websocket is only closed on a normal stream end.
+        """
 
-                payload_bytes = result.value.bytes_ if result.value else None
-                if payload_bytes:
-                    bedrock_response = payload_bytes.decode("utf-8")
-                    verbose_proxy_logger.debug("Bedrock Realtime: Received from Bedrock: %s", bedrock_response[:200])
-
-                    # Transform Bedrock format to OpenAI format
-                    realtime_response_transform_input: RealtimeResponseTransformInput = {
-                        "current_output_item_id": session_state.get("current_output_item_id"),
-                        "current_response_id": session_state.get("current_response_id"),
-                        "current_conversation_id": session_state.get("current_conversation_id"),
-                        "current_delta_chunks": session_state.get("current_delta_chunks"),
-                        "current_item_chunks": session_state.get("current_item_chunks"),
-                        "current_delta_type": session_state.get("current_delta_type"),
-                        "session_configuration_request": session_state.get("session_configuration_request"),
-                    }
-
-                    transformed_response = transformation_config.transform_realtime_response(
-                        message=bedrock_response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        realtime_response_transform_input=realtime_response_transform_input,
-                    )
-
-                    # Update session state
-                    session_state.update(
-                        {
-                            "current_output_item_id": transformed_response.get("current_output_item_id"),
-                            "current_response_id": transformed_response.get("current_response_id"),
-                            "current_conversation_id": transformed_response.get("current_conversation_id"),
-                            "current_delta_chunks": transformed_response.get("current_delta_chunks"),
-                            "current_item_chunks": transformed_response.get("current_item_chunks"),
-                            "current_delta_type": transformed_response.get("current_delta_type"),
-                            "session_configuration_request": transformed_response.get("session_configuration_request"),
-                        }
-                    )
-
-                    # Send transformed messages to client
-                    response_value = transformed_response["response"]
-                    openai_messages = response_value if isinstance(response_value, list) else (response_value,)
-                    for openai_message in openai_messages:
-                        message_json = json.dumps(openai_message)
-                        await client_ws.send_text(message_json)
-                        verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
-                        if _should_log_event(openai_message):
-                            yield openai_message
-
-        except Exception as e:
-            verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True)
-        finally:
-            # Close the client WebSocket
+        async def send_to_client(message_json: str) -> bool:
             try:
-                await client_ws.close()
-            except Exception:
-                pass
+                await client_ws.send_text(message_json)
+            except Exception as e:  # noqa: BLE001  # any send failure means the client is gone
+                verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True)
+                return False
+            verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
+            return True
+
+        output: Final = await bedrock_stream.await_output()
+        while True:
+            result = await output[1].receive()
+
+            if result is None:
+                verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
+                with contextlib.suppress(Exception):
+                    await client_ws.close()
+                return
+
+            payload_bytes = result.value.bytes_ if result.value else None
+            if payload_bytes:
+                bedrock_response = payload_bytes.decode("utf-8")
+                verbose_proxy_logger.debug("Bedrock Realtime: Received from Bedrock: %s", bedrock_response[:200])
+
+                # Transform Bedrock format to OpenAI format
+                realtime_response_transform_input: RealtimeResponseTransformInput = {
+                    "current_output_item_id": session_state.get("current_output_item_id"),
+                    "current_response_id": session_state.get("current_response_id"),
+                    "current_conversation_id": session_state.get("current_conversation_id"),
+                    "current_delta_chunks": session_state.get("current_delta_chunks"),
+                    "current_item_chunks": session_state.get("current_item_chunks"),
+                    "current_delta_type": session_state.get("current_delta_type"),
+                    "session_configuration_request": session_state.get("session_configuration_request"),
+                }
+
+                transformed_response = transformation_config.transform_realtime_response(
+                    message=bedrock_response,
+                    model=model,
+                    logging_obj=logging_obj,
+                    realtime_response_transform_input=realtime_response_transform_input,
+                )
+
+                # Update session state
+                session_state.update(
+                    {
+                        "current_output_item_id": transformed_response.get("current_output_item_id"),
+                        "current_response_id": transformed_response.get("current_response_id"),
+                        "current_conversation_id": transformed_response.get("current_conversation_id"),
+                        "current_delta_chunks": transformed_response.get("current_delta_chunks"),
+                        "current_item_chunks": transformed_response.get("current_item_chunks"),
+                        "current_delta_type": transformed_response.get("current_delta_type"),
+                        "session_configuration_request": transformed_response.get("session_configuration_request"),
+                    }
+                )
+
+                # Send transformed messages to client
+                response_value = transformed_response["response"]
+                openai_messages = response_value if isinstance(response_value, list) else (response_value,)
+                for openai_message in openai_messages:
+                    if not await send_to_client(json.dumps(openai_message)):
+                        return
+                    if _should_log_event(openai_message):
+                        yield openai_message

litellm/llms/bedrock_mantle/chat/transformation.py

@@ -29,7 +29,7 @@ from litellm.types.router import GenericLiteLLMParams
 from ...base_llm.chat.transformation import BaseLLMException
 from ...bedrock.common_utils import BedrockError
 from ...openai_like.chat.transformation import OpenAILikeChatConfig
-from ..common_utils import mantle_base_segment
+from ..common_utils import mantle_base_segment, split_mantle_region_prefix
 
 
 class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
@@ -61,8 +61,10 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
         litellm_params: GenericLiteLLMParams | None = None,
         model: str | None = None,
     ) -> tuple[str | None, str | None]:
+        prefix_region, base_model = split_mantle_region_prefix(model) if model else (None, None)
         region: Final = (
             (litellm_params.aws_region_name if litellm_params else None)
+            or prefix_region
             or get_secret_str("BEDROCK_MANTLE_REGION")
             or get_secret_str("AWS_REGION_NAME")
             or get_secret_str("AWS_REGION")
@@ -75,7 +77,7 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
         api_base = (
             api_base
             or get_secret_str("BEDROCK_MANTLE_API_BASE")
-            or f"https://bedrock-mantle.{region}.api.aws/{mantle_base_segment(model, litellm.model_cost)}"
+            or f"https://bedrock-mantle.{region}.api.aws/{mantle_base_segment(base_model, litellm.model_cost)}"
         )
         dynamic_api_key: Final = self._resolve_bearer_token(api_key)
         return api_base, dynamic_api_key

litellm/llms/bedrock_mantle/common_utils.py

@@ -24,9 +24,11 @@ from botocore.exceptions import (
 )
 
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS
+from litellm.llms.bedrock.common_utils import AmazonBedrockGlobalConfig
 from litellm.secret_managers.main import get_secret_str
 
 BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1"
+BEDROCK_REGIONS: Final = frozenset(AmazonBedrockGlobalConfig().get_all_regions())
 
 # Standard Mantle host: https://bedrock-mantle.<region>.api.aws (group 1 = region).
 MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws(?=/|$)", re.IGNORECASE)
@@ -36,6 +38,13 @@ def resolve_mantle_bearer_token(api_key: str | None) -> str | None:
     return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
 
 
+def split_mantle_region_prefix(model: str) -> tuple[str | None, str]:
+    head, sep, tail = model.partition("/")
+    if sep and head in BEDROCK_REGIONS:
+        return head, tail
+    return None, model
+
+
 def resolve_mantle_region(params: Mapping[str, object]) -> str:
     region: Final = params.get("aws_region_name")
     if isinstance(region, str) and region:
@@ -130,7 +139,7 @@ def mantle_supports_responses(model: str | None, model_cost: dict) -> bool:
     gpt-oss substring), so a substring gate would be wrong. A model absent from
     model_cost simply has no signal and returns False (chat-completions emulation).
     """
-    entry: Final = model_cost.get(f"bedrock_mantle/{model}", {})
+    entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}", {}) if model else {}
     if "/v1/responses" in (entry.get("supported_endpoints") or []):
         return True
     return entry.get("mode") == "responses"
@@ -147,5 +156,5 @@ def mantle_base_segment(model: str | None, model_cost: dict) -> str:
     the base for the model's whole OpenAI-compatible surface, so both the chat and
     responses configs derive from it -- there is no separate model-name rule.
     """
-    entry: Final = model_cost.get(f"bedrock_mantle/{model}", {})
+    entry: Final = model_cost.get(f"bedrock_mantle/{split_mantle_region_prefix(model)[1]}", {}) if model else {}
     return "openai/v1" if entry.get("use_openai_responses_path") is True else "v1"

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -17,7 +17,7 @@ BaseAWSLLM._sign_request after the request body is finalized.
 
 import json
 from collections.abc import Mapping
-from typing import Any, Final
+from typing import Any, Final, cast  # noqa: TID251  # map_openai_params returns the filtered params as a bare dict
 
 import httpx
 from typing_extensions import ReadOnly, TypedDict
@@ -32,6 +32,7 @@ from litellm.llms.bedrock_mantle.common_utils import (
     BedrockMantleAuthMixin,
 )
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
     ResponseInputParam,
@@ -58,8 +59,6 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset(
 _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 _BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
 
-_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
-
 _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
 _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
 _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
@@ -233,17 +232,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         litellm_params: GenericLiteLLMParams,
         headers: dict,
     ) -> dict:
-        remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
-        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
+        params: Final = cast(  # cast-ok: the base signature leaves the params dict untyped
+            "ResponsesAPIOptionalRequestParams", response_api_optional_request_params
+        )
+        hoisted: Final = hoist_additional_tools(input, params.get("tools"))
+        normalized_input: Final = self._normalize_codex_input_items(hoisted.input)
         request_params: Final = (
-            {
-                **response_api_optional_request_params,
-                "tools": [
-                    *(response_api_optional_request_params.get("tools") or []),
-                    *hoisted_tools,
-                ],
-            }
-            if hoisted_tools
+            self._params_with_hoisted_tools(params, hoisted)
+            if hoisted.hoisted
             else response_api_optional_request_params
         )
         return super().transform_responses_api_request(
@@ -254,41 +250,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
             headers=headers,
         )
 
-    @staticmethod
-    def _is_codex_additional_tools_item(item: Any) -> bool:
-        return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
-
-    @staticmethod
-    def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]":
-        tools: Final = item.get("tools")
-        return tools if isinstance(tools, list) else []
-
     @classmethod
-    def _hoist_codex_additional_tools(
-        cls,
-        input: "str | ResponseInputParam",
-    ) -> "tuple[str | ResponseInputParam, list[Any]]":
-        """Codex's "responses lite" wire mode ships tool definitions inside
-        `input` as {"type": "additional_tools", "role": "developer",
-        "tools": [...]} items. api.openai.com accepts that item type; Mantle
-        rejects the whole request with 400 "Invalid 'input': value did not
-        match any expected variant" but accepts the same tools at the top
-        level, so move them there and strip the items from `input`.
-        """
-        if not isinstance(input, list):
-            return input, []
-        additional_tools_items: Final = [item for item in input if cls._is_codex_additional_tools_item(item)]
-        if not additional_tools_items:
-            return input, []
-        remaining_input: Final = [item for item in input if not cls._is_codex_additional_tools_item(item)]
-        hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)]
-        verbose_logger.debug(
-            "Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) "
-            "into the top-level tools param (Mantle rejects that input item type).",
-            len(hoisted_tools),
-            len(additional_tools_items),
-        )
-        return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
+    def _params_with_hoisted_tools(
+        cls, params: Mapping[str, object], hoisted: HoistedAdditionalTools
+    ) -> dict[str, object]:
+        supported_tools: Final = cls._filter_unsupported_tools(list(hoisted.tools))
+        if supported_tools:
+            return {**params, "tools": supported_tools}
+        return {key: value for key, value in params.items() if key != "tools"}
 
     @staticmethod
     def _agent_message_text(item: "Mapping[str, object]") -> str:

litellm/llms/custom_httpx/llm_http_handler.py

@@ -18,7 +18,7 @@ import litellm.types
 import litellm.types.utils
 from litellm._logging import _redact_string, verbose_logger
 from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
-from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
+from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.litellm_core_utils.agentic_loop_settings import (
     DEFAULT_MAX_AGENTIC_LOOPS,
     validated_max_agentic_loops,
@@ -4981,15 +4981,16 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            response: Final = sync_httpx_client.get(url=url, headers=headers, params=params)
+            response: Final = sync_httpx_client.get(url=url, headers=headers, params=params, timeout=timeout)
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
 
-        return provider_config.transform_list_files_response(
-            raw_response=response,
-            logging_obj=logging_obj,
-            litellm_params=litellm_params,
+        files_per_page: Final = self._files_per_listing_page(
+            response, provider_config, logging_obj, litellm_params, headers, sync_httpx_client, timeout
         )
+        return [  # mutable-ok: the files contract returns the listing as a list
+            listed_file for page_files in files_per_page for listed_file in page_files
+        ]
 
     async def async_list_files(
         self,
@@ -5037,16 +5038,101 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            response: Final = await async_httpx_client.get(url=url, headers=headers, params=params)
+            response: Final = await async_httpx_client.get(url=url, headers=headers, params=params, timeout=timeout)
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
 
-        return provider_config.transform_list_files_response(
-            raw_response=response,
-            logging_obj=logging_obj,
+        files_per_page: Final = self._files_per_async_listing_page(
+            response, provider_config, logging_obj, litellm_params, headers, async_httpx_client, timeout
+        )
+        return [  # mutable-ok: the files contract returns the listing as a list
+            listed_file async for page_files in files_per_page for listed_file in page_files
+        ]
+
+    def _files_per_listing_page(
+        self,
+        first_page: httpx.Response,
+        provider_config: BaseFilesConfig,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: dict,  # mutable-ok: handed to the files contract, which types it as a dict
+        headers: dict,  # mutable-ok: handed to validate_environment, which types it as a dict
+        client: HTTPHandler,
+        timeout: float | httpx.Timeout | None,
+    ) -> Iterator[list[OpenAIFileObject]]:  # mutable-ok: each page arrives as the list the files contract returns
+        latest_page = first_page  # rebind-ok: advances one page per loop turn
+        listed_count = 0  # rebind-ok: grows per page so the listing stops at MAX_FILE_LIST_LIMIT, OpenAI's ceiling
+        while True:
+            page_files = provider_config.transform_list_files_response(
+                raw_response=latest_page, logging_obj=logging_obj, litellm_params=litellm_params
+            )
+            yield page_files[: MAX_FILE_LIST_LIMIT - listed_count]
+            listed_count += len(page_files)
+            next_request = self._next_listing_request(latest_page, provider_config, litellm_params, listed_count)
+            if next_request is None:
+                return
+            url, params = next_request
+            next_headers = self._next_listing_page_headers(provider_config, headers, litellm_params)
+            try:
+                latest_page = client.get(url=url, headers=next_headers, params=params, timeout=timeout)
+            except Exception as e:  # noqa: BLE001  # _handle_error maps every failure kind, like the first page's fetch
+                raise self._handle_error(e=e, provider_config=provider_config)
+
+    async def _files_per_async_listing_page(
+        self,
+        first_page: httpx.Response,
+        provider_config: BaseFilesConfig,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: dict,  # mutable-ok: handed to the files contract, which types it as a dict
+        headers: dict,  # mutable-ok: handed to validate_environment, which types it as a dict
+        client: AsyncHTTPHandler,
+        timeout: float | httpx.Timeout | None,
+    ) -> AsyncIterator[list[OpenAIFileObject]]:  # mutable-ok: each page arrives as the list the files contract returns
+        latest_page = first_page  # rebind-ok: advances one page per loop turn
+        listed_count = 0  # rebind-ok: grows per page so the listing stops at MAX_FILE_LIST_LIMIT, OpenAI's ceiling
+        while True:
+            page_files = provider_config.transform_list_files_response(
+                raw_response=latest_page, logging_obj=logging_obj, litellm_params=litellm_params
+            )
+            yield page_files[: MAX_FILE_LIST_LIMIT - listed_count]
+            listed_count += len(page_files)
+            next_request = self._next_listing_request(latest_page, provider_config, litellm_params, listed_count)
+            if next_request is None:
+                return
+            url, params = next_request
+            next_headers = self._next_listing_page_headers(provider_config, headers, litellm_params)
+            try:
+                latest_page = await client.get(url=url, headers=next_headers, params=params, timeout=timeout)
+            except Exception as e:  # noqa: BLE001  # _handle_error maps every failure kind, like the first page's fetch
+                raise self._handle_error(e=e, provider_config=provider_config)
+
+    def _next_listing_page_headers(
+        self,
+        provider_config: BaseFilesConfig,
+        headers: dict,  # mutable-ok: handed to validate_environment, which types it as a dict
+        litellm_params: dict,  # mutable-ok: handed to the files contract, which types it as a dict
+    ) -> dict:  # mutable-ok: validate_environment returns the header dict the files contract types
+        return provider_config.validate_environment(
+            api_key=litellm_params.get("api_key"),
+            headers=headers,
+            model="",
+            messages=[],
+            optional_params={},
             litellm_params=litellm_params,
         )
 
+    def _next_listing_request(
+        self,
+        latest_page: httpx.Response,
+        provider_config: BaseFilesConfig,
+        litellm_params: dict,  # mutable-ok: handed to the files contract, which types it as a dict
+        listed_count: int,
+    ) -> tuple[str, dict[str, str]] | None:  # mutable-ok: the base files contract returns the query as a dict
+        if listed_count >= MAX_FILE_LIST_LIMIT:
+            return None
+        return provider_config.transform_list_files_next_request(
+            raw_response=latest_page, optional_params={}, litellm_params=litellm_params
+        )
+
     def retrieve_file_content(
         self,
         file_content_request: "FileContentRequest",

litellm/llms/databricks/chat/transformation.py

@@ -272,6 +272,15 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             "thinking",
         ]
 
+    @staticmethod
+    def _uses_anthropic_thinking_param(model: str) -> bool:
+        from litellm.utils import supports_anthropic_thinking_payload
+
+        normalized: Final = model.lower().replace(".", "-")
+        return "claude" in normalized or supports_anthropic_thinking_payload(
+            model=normalized, custom_llm_provider="databricks"
+        )
+
     def convert_anthropic_tool_to_databricks_tool(self, tool: AllAnthropicToolsValues | None) -> DatabricksTool | None:
         if tool is None:
             return None
@@ -377,7 +386,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
                 "response_format", None
             )  # unsupported for claude models - if json_schema -> convert to tool call
 
-        if "reasoning_effort" in non_default_params and "claude" in model:
+        if "reasoning_effort" in non_default_params and self._uses_anthropic_thinking_param(model):
             reasoning_effort_value: Final = non_default_params.get("reasoning_effort")
             mapped_thinking: Final = AnthropicConfig._map_reasoning_effort(
                 reasoning_effort=reasoning_effort_value,

litellm/llms/fireworks_ai/chat/transformation.py

@@ -602,6 +602,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
             return None
         return max(matches, key=lambda match: len(match[0]))[1]
 
+    def get_model_cost_key(self, model: str) -> str:
+        return f"fireworks_ai/{resolve_fireworks_resource_name(model)}"
+
     def get_provider_info(self, model: str) -> ProviderSpecificModelInfo:
         supports_function_calling_value: Final = self._get_model_cost_capability(
             model=model, capability="supports_function_calling"

litellm/llms/gemini/realtime/transformation.py

@@ -115,6 +115,19 @@ def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup
     return envelope.get("setup", empty_setup)
 
 
+def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
+    """Read ``serverContent.groundingMetadata`` off the frame that carries the turn's usage.
+
+    Live reports grounding in the server frames rather than in ``usageMetadata``, and it emits both
+    on the same frame, so the per-query charge is countable at the point usage is built.
+    """
+    server_content: Final = frame.get("serverContent")
+    if not isinstance(server_content, Mapping):
+        return ()
+    metadata: Final = server_content.get("groundingMetadata")
+    return (metadata,) if isinstance(metadata, Mapping) else ()
+
+
 # Google bills Live transcription at an estimated 25 audio tokens/sec of input and
 # 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
 GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
@@ -323,7 +336,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 )
             elif key == "input_audio_transcription" and value is not None:
                 optional_params["inputAudioTranscription"] = {}
-            elif key == "turn_detection":
+            elif key == "turn_detection" and value is not None:
                 value_typed = cast(OpenAIRealtimeTurnDetection, value)
                 if (
                     isinstance(value_typed, dict)
@@ -1049,6 +1062,11 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                     {**cast(dict, message), "usageMetadata": resolved_usage_metadata},
                 ),
             )
+            grounding_metadata: Final = _grounding_metadata_from_frame(message)
+            if grounding_metadata:
+                VertexGeminiConfig._set_grounding_usage_counters(  # pyright: ignore[reportPrivateUsage]  # shared with the chat path; no public alias exists yet
+                    _chat_completion_usage, grounding_metadata
+                )
         else:
             _chat_completion_usage = get_empty_usage()
 

litellm/llms/meta/realtime/transformation.py

@@ -0,0 +1,719 @@
+import asyncio
+import base64
+import binascii
+import json
+import math
+import time
+from collections.abc import Awaitable, Callable, Iterator, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final, Literal
+from urllib.parse import urlparse, urlunparse
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm import verbose_logger
+from litellm._uuid import uuid
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.meta import MuseAudioEncoding, MuseHandshake, MuseMode, MuseSampleRate
+from litellm.types.llms.openai import (
+    OpenAIRealtimeErrorEvent,
+    OpenAIRealtimeEvents,
+    OpenAIRealtimeInputAudioBufferSpeechEvent,
+    OpenAIRealtimeInputAudioTranscriptionCompleted,
+    OpenAIRealtimeInputAudioTranscriptionDelta,
+    OpenAIRealtimeServerVadTurnDetection,
+    OpenAIRealtimeTranscriptionSession,
+    OpenAIRealtimeTranscriptionSessionCreated,
+    OpenAIRealtimeTranscriptionSettings,
+)
+from litellm.types.realtime import (
+    RealtimeInputAudioTranscriptionDurationUsage,
+    RealtimeInputAudioTranscriptionUsage,
+    RealtimeResponseTransformInput,
+    RealtimeResponseTypedDict,
+)
+
+MUSE_MODEL: Final = "muse-voice-transcribe-1.0"
+DEFAULT_MUSE_REALTIME_URL: Final = "wss://api.meta.ai/v1/asr/realtime"
+SUPPORTED_SAMPLE_RATES: Final = frozenset((16_000, 24_000))
+SUPPORTED_LANGUAGES: Final = (
+    "Arabic",
+    "Bengali",
+    "Dutch",
+    "English",
+    "French",
+    "German",
+    "Hebrew",
+    "Hindi",
+    "Indonesian",
+    "Italian",
+    "Japanese",
+    "Kannada",
+    "Korean",
+    "Malay",
+    "Mandarin Chinese",
+    "Marathi",
+    "Polish",
+    "Portuguese",
+    "Spanish",
+    "Tagalog",
+    "Tamil",
+    "Telugu",
+    "Thai",
+    "Turkish",
+    "Vietnamese",
+)
+_LANGUAGE_NAMES: Final = MappingProxyType({language.casefold(): language for language in SUPPORTED_LANGUAGES})
+_LANGUAGE_CODES: Final = MappingProxyType(
+    {
+        "ar": "Arabic",
+        "bn": "Bengali",
+        "de": "German",
+        "en": "English",
+        "es": "Spanish",
+        "fil": "Tagalog",
+        "fr": "French",
+        "he": "Hebrew",
+        "hi": "Hindi",
+        "id": "Indonesian",
+        "it": "Italian",
+        "iw": "Hebrew",
+        "ja": "Japanese",
+        "kn": "Kannada",
+        "ko": "Korean",
+        "ms": "Malay",
+        "mr": "Marathi",
+        "nl": "Dutch",
+        "pl": "Polish",
+        "pt": "Portuguese",
+        "ta": "Tamil",
+        "te": "Telugu",
+        "th": "Thai",
+        "tl": "Tagalog",
+        "tr": "Turkish",
+        "vi": "Vietnamese",
+        "zh": "Mandarin Chinese",
+    }
+)
+_SUPPORTED_TRANSCRIPTION_KEYS: Final = frozenset(("model", "language"))
+_MAX_AUDIO_BACKLOG_SECONDS: Final = 4
+_PACKET_MS: Final = 80
+_END_STREAM: Final = '{"type":"endStream"}'
+_PROVIDER_ERROR_MESSAGE: Final = "Meta Muse realtime transcription failed"
+_JSON_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+_EMPTY_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
+_SERVER_VAD: Final[OpenAIRealtimeServerVadTurnDetection] = {"type": "server_vad"}
+
+
+class MuseProtocolError(ValueError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class MuseSessionConfig:
+    model: str
+    mode: MuseMode
+    sample_rate: MuseSampleRate
+    language_bias: tuple[str, ...]
+
+    @property
+    def audio_encoding(self) -> MuseAudioEncoding:
+        return "PCM_16KHZ" if self.sample_rate == 16_000 else "PCM_24KHZ"
+
+    @property
+    def bytes_per_second(self) -> int:
+        return self.sample_rate * 2
+
+    @property
+    def packet_bytes(self) -> int:
+        return self.bytes_per_second * _PACKET_MS // 1000
+
+    @property
+    def max_encoded_append_bytes(self) -> int:
+        return 4 * ((self.bytes_per_second * _MAX_AUDIO_BACKLOG_SECONDS + 2) // 3)
+
+    def handshake(self, access_token: str) -> MuseHandshake:
+        base: Final[MuseHandshake] = {
+            "authorization": {"accessToken": access_token},
+            "audioEncoding": self.audio_encoding,
+            "model": self.model,
+            "mode": self.mode,
+            "partialMode": "CUMULATIVE",
+            "emitAudioProgress": True,
+        }
+        if not self.language_bias:
+            return base
+        biased: Final[MuseHandshake] = {**base, "languageBias": self.language_bias}
+        return biased
+
+    def openai_session(self, session_id: str) -> OpenAIRealtimeTranscriptionSession:
+        session: Final[OpenAIRealtimeTranscriptionSession] = {
+            "id": session_id,
+            "object": "realtime.transcription_session",
+            "type": "transcription",
+            "audio": {
+                "input": {
+                    "format": {"type": "audio/pcm", "rate": self.sample_rate},
+                    "transcription": self._transcription_settings(),
+                    "turn_detection": None if self.mode == "PUSH_TO_TALK" else _SERVER_VAD,
+                }
+            },
+        }
+        return session
+
+    def _transcription_settings(self) -> OpenAIRealtimeTranscriptionSettings:
+        base: Final[OpenAIRealtimeTranscriptionSettings] = {"model": self.model}
+        if not self.language_bias:
+            return base
+        localized: Final[OpenAIRealtimeTranscriptionSettings] = {**base, "language": self.language_bias[0]}
+        return localized
+
+
+_DEFAULT_SESSION_CONFIG: Final = MuseSessionConfig(
+    model=MUSE_MODEL, mode="ENDPOINTING", sample_rate=24_000, language_bias=()
+)
+
+
+def _json_object(payload: str) -> Mapping[str, JsonValue]:
+    try:
+        value: Final = _JSON_ADAPTER.validate_json(payload)
+    except ValidationError:
+        raise MuseProtocolError("invalid JSON object") from None
+    if not isinstance(value, dict):
+        raise MuseProtocolError("message must be a JSON object")
+    return value
+
+
+def _mapping(value: JsonValue | None, name: str) -> Mapping[str, JsonValue]:
+    if value is None:
+        return _EMPTY_OBJECT
+    if not isinstance(value, dict):
+        raise MuseProtocolError(f"{name} must be an object")
+    return value
+
+
+def _string(value: JsonValue | None, name: str) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise MuseProtocolError(f"{name} must be a string")
+    return value
+
+
+def _normalize_model(model: str) -> str:
+    return model.removeprefix("meta/").strip()
+
+
+def _event_id() -> str:
+    return f"event_{uuid.uuid4().hex}"
+
+
+def normalize_language(language: str) -> str:
+    value: Final = language.strip()
+    if not value:
+        raise MuseProtocolError("language must be non-empty")
+    documented_name: Final = _LANGUAGE_NAMES.get(value.casefold())
+    if documented_name is not None:
+        return documented_name
+    primary: Final = value.replace("_", "-").split("-", 1)[0].casefold()
+    mapped_name: Final = _LANGUAGE_CODES.get(primary)
+    if mapped_name is None:
+        raise MuseProtocolError("unsupported Muse Voice language")
+    return mapped_name
+
+
+def normalize_access_token(api_key: str) -> str:
+    stripped: Final = api_key.strip()
+    if not stripped:
+        raise ValueError("Meta API key is required")
+    parts: Final = stripped.split(None, 1)
+    if parts[0].casefold() != "bearer":
+        return f"Bearer {stripped}"
+    if len(parts) != 2 or not parts[1].strip():
+        raise ValueError("Meta API key must include a token after Bearer")
+    return f"Bearer {parts[1].strip()}"
+
+
+def build_muse_realtime_url(api_base: str | None) -> str:
+    if api_base is None:
+        return DEFAULT_MUSE_REALTIME_URL
+    parsed: Final = urlparse(api_base.strip())
+    scheme: Final = "wss" if parsed.scheme == "https" else parsed.scheme
+    if (
+        scheme != "wss"
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.fragment
+    ):
+        raise ValueError("Meta api_base must be an absolute wss:// or https:// URL without credentials or a fragment")
+    netloc: Final = f"{parsed.hostname}:{parsed.port}" if parsed.port is not None else parsed.hostname
+    return urlunparse((scheme, netloc, "/v1/asr/realtime", "", "", ""))
+
+
+def _parse_sample_rate(session: Mapping[str, JsonValue]) -> MuseSampleRate:
+    beta_format: Final = session.get("input_audio_format")
+    audio: Final = _mapping(session.get("audio"), "session.audio")
+    audio_input: Final = _mapping(audio.get("input"), "session.audio.input")
+    ga_format: Final = audio_input.get("format")
+    if beta_format is not None and ga_format is not None:
+        raise MuseProtocolError("input audio format must use either beta or GA layout")
+    if beta_format is not None:
+        if beta_format != "pcm16":
+            raise MuseProtocolError("Muse Voice requires pcm16 input audio")
+        return 24_000
+    if ga_format is None:
+        return 24_000
+    if isinstance(ga_format, str):
+        if ga_format != "pcm16":
+            raise MuseProtocolError("Muse Voice requires audio/pcm input audio")
+        return 24_000
+    format_mapping: Final = _mapping(ga_format, "session.audio.input.format")
+    if format_mapping.get("type") != "audio/pcm":
+        raise MuseProtocolError("Muse Voice requires audio/pcm input audio")
+    channels: Final = format_mapping.get("channels", 1)
+    if isinstance(channels, bool) or channels != 1:
+        raise MuseProtocolError("Muse Voice requires mono input audio")
+    rate: Final = format_mapping.get("rate", 24_000)
+    if isinstance(rate, bool) or not isinstance(rate, int) or rate not in SUPPORTED_SAMPLE_RATES:
+        raise MuseProtocolError("Muse Voice supports PCM16 at 16000 Hz or 24000 Hz")
+    return 16_000 if rate == 16_000 else 24_000
+
+
+def _parse_mode(session: Mapping[str, JsonValue], audio_input: Mapping[str, JsonValue]) -> MuseMode:
+    turn_detection_present: Final = "turn_detection" in session or "turn_detection" in audio_input
+    turn_detection: Final = session.get("turn_detection", audio_input.get("turn_detection"))
+    if turn_detection_present and turn_detection is None:
+        return "PUSH_TO_TALK"
+    if turn_detection is None:
+        return "ENDPOINTING"
+    turn_detection_mapping: Final = _mapping(turn_detection, "turn_detection")
+    if turn_detection_mapping.get("type") not in (None, "server_vad"):
+        raise MuseProtocolError("Muse Voice supports server_vad turn detection or null")
+    return "ENDPOINTING"
+
+
+def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig:
+    message: Final = _json_object(payload)
+    if message.get("type") not in ("session.update", "transcription_session.update"):
+        raise MuseProtocolError("expected session.update")
+    session: Final = _mapping(message.get("session"), "session")
+    if not session:
+        raise MuseProtocolError("session.update requires a session object")
+    if session.get("type") not in (None, "transcription", "realtime"):
+        raise MuseProtocolError("Muse Voice supports transcription sessions only")
+    audio: Final = _mapping(session.get("audio"), "session.audio")
+    audio_input: Final = _mapping(audio.get("input"), "session.audio.input")
+    beta_transcription: Final = session.get("input_audio_transcription")
+    ga_transcription: Final = audio_input.get("transcription")
+    if beta_transcription is not None and ga_transcription is not None:
+        raise MuseProtocolError("input transcription must use either beta or GA layout")
+    transcription: Final = _mapping(
+        beta_transcription if beta_transcription is not None else ga_transcription,
+        "input audio transcription",
+    )
+    unsupported: Final = tuple(sorted(key for key in transcription if key not in _SUPPORTED_TRANSCRIPTION_KEYS))
+    if unsupported:
+        verbose_logger.warning("Meta realtime: dropping unsupported transcription settings %s", unsupported)
+    requested_model: Final = _string(transcription.get("model"), "transcription model")
+    normalized_model: Final = _normalize_model(expected_model)
+    if normalized_model != MUSE_MODEL:
+        raise MuseProtocolError("unsupported Meta realtime model")
+    if requested_model is not None and _normalize_model(requested_model) != normalized_model:
+        raise MuseProtocolError("realtime session model cannot be changed")
+    language: Final = _string(transcription.get("language"), "language")
+    return MuseSessionConfig(
+        model=normalized_model,
+        mode=_parse_mode(session, audio_input),
+        sample_rate=_parse_sample_rate(session),
+        language_bias=() if language is None else (normalize_language(language),),
+    )
+
+
+def session_created_event(config: MuseSessionConfig, session_id: str) -> OpenAIRealtimeTranscriptionSessionCreated:
+    event: Final[OpenAIRealtimeTranscriptionSessionCreated] = {
+        "type": "session.created",
+        "event_id": _event_id(),
+        "session": config.openai_session(session_id),
+    }
+    return event
+
+
+def error_event(message: str) -> OpenAIRealtimeErrorEvent:
+    event: Final[OpenAIRealtimeErrorEvent] = {
+        "type": "error",
+        "error": {"type": "server_error", "message": message},
+    }
+    return event
+
+
+def _speech_event(
+    event_type: Literal["input_audio_buffer.speech_started", "input_audio_buffer.speech_stopped"], item_id: str
+) -> OpenAIRealtimeInputAudioBufferSpeechEvent:
+    event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = {
+        "type": event_type,
+        "event_id": _event_id(),
+        "item_id": item_id,
+    }
+    return event
+
+
+def _delta_event(item_id: str, delta: str) -> OpenAIRealtimeInputAudioTranscriptionDelta:
+    event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = {
+        "type": "conversation.item.input_audio_transcription.delta",
+        "event_id": _event_id(),
+        "item_id": item_id,
+        "content_index": 0,
+        "delta": delta,
+    }
+    return event
+
+
+def _completed_event(
+    item_id: str, transcript: str, usage: RealtimeInputAudioTranscriptionUsage | None
+) -> OpenAIRealtimeInputAudioTranscriptionCompleted:
+    event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = {
+        "type": "conversation.item.input_audio_transcription.completed",
+        "event_id": _event_id(),
+        "item_id": item_id,
+        "content_index": 0,
+        "transcript": transcript,
+    }
+    if usage is None:
+        return event
+    billed: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = {**event, "usage": usage}
+    return billed
+
+
+def _required_turn_id(message: Mapping[str, JsonValue], event: str) -> str:
+    value: Final = message.get("turnId")
+    if isinstance(value, bool) or not isinstance(value, (str, int)):
+        raise MuseProtocolError(f"{event} event has invalid turnId")
+    turn_id: Final = str(value).strip()
+    if not turn_id:
+        raise MuseProtocolError(f"{event} event has invalid turnId")
+    return turn_id
+
+
+def _new_suffix(previous: str, current: str) -> str:
+    return current[len(previous) :] if current.startswith(previous) else ""
+
+
+@dataclass(slots=True)
+class _TurnState:
+    item_id: str
+    started: bool = False
+    start_emitted: bool = False
+    latest_partial: str | None = None
+    emitted_partial: str = ""
+    final_text: str | None = None
+    completed_emitted: bool = False
+    stopped: bool = False
+    stopped_emitted: bool = False
+
+    def finish(self, transcript: str) -> None:
+        self.final_text = transcript
+        self.stopped = True
+
+    def drain(
+        self, take_usage: Callable[[], RealtimeInputAudioTranscriptionUsage | None]
+    ) -> Iterator[OpenAIRealtimeEvents]:
+        has_content: Final = self.latest_partial is not None or self.final_text is not None
+        if (self.started or has_content) and not self.start_emitted:
+            self.start_emitted = True
+            yield _speech_event("input_audio_buffer.speech_started", self.item_id)
+        if self.latest_partial is not None and self.final_text is None:
+            delta: Final = _new_suffix(self.emitted_partial, self.latest_partial)
+            if delta:
+                self.emitted_partial = self.latest_partial
+                yield _delta_event(self.item_id, delta)
+        if self.stopped and not self.stopped_emitted:
+            self.stopped_emitted = True
+            yield _speech_event("input_audio_buffer.speech_stopped", self.item_id)
+        if self.final_text is not None and self.stopped_emitted and not self.completed_emitted:
+            self.completed_emitted = True
+            yield _completed_event(self.item_id, self.final_text, take_usage())
+
+
+class MuseEventTransformer:
+    def __init__(self, *, turn_limit: int = 128) -> None:
+        self._turns: dict[str, _TurnState] = {}  # mutable-ok: bounded, insertion-ordered per-turn emit state
+        self._turn_limit: Final = turn_limit
+        self._active_turn_id: str | None = None
+        self._mode: MuseMode = "ENDPOINTING"
+        self._last_audio_processed_ms: float = 0.0
+        self._unbilled_seconds: float = 0.0
+
+    def configure(self, config: MuseSessionConfig) -> None:
+        self._mode = config.mode
+
+    def transform(self, message: Mapping[str, JsonValue]) -> tuple[OpenAIRealtimeEvents, ...]:
+        event_type: Final = message.get("type")
+        if event_type == "error":
+            return (error_event(_PROVIDER_ERROR_MESSAGE),)
+        if event_type == "audioProgress":
+            self._update_audio_progress(message)
+            return ()
+        turn: Final = self._apply_turn_event(event_type, message)
+        if turn is None:
+            return ()
+        return tuple(turn.drain(self.take_unbilled_usage))
+
+    def take_unbilled_usage(self) -> RealtimeInputAudioTranscriptionUsage | None:
+        seconds: Final = self._unbilled_seconds
+        if seconds <= 0:
+            return None
+        self._unbilled_seconds = 0.0
+        usage: Final[RealtimeInputAudioTranscriptionDurationUsage] = {"type": "duration", "seconds": seconds}
+        return usage
+
+    def _apply_turn_event(self, event_type: JsonValue | None, message: Mapping[str, JsonValue]) -> _TurnState | None:
+        match event_type:
+            case "speechStart":
+                return self._speech_start(message)
+            case "transcript":
+                return self._transcript(message)
+            case "speechEnd":
+                return self._speech_end(message)
+            case "speechComplete":
+                return self._speech_complete(message)
+            case _:
+                return None
+
+    def _turn(self, turn_id: str) -> _TurnState:
+        existing: Final = self._turns.get(turn_id)
+        if existing is not None:
+            return existing
+        created: Final = _TurnState(item_id=turn_id)
+        self._turns[turn_id] = created
+        if len(self._turns) > self._turn_limit:
+            del self._turns[next(iter(self._turns))]
+        return created
+
+    def _speech_start(self, message: Mapping[str, JsonValue]) -> _TurnState:
+        turn: Final = self._turn(_required_turn_id(message, "speechStart"))
+        if turn.stopped:
+            return turn
+        turn.started = True
+        self._active_turn_id = turn.item_id
+        return turn
+
+    def _transcript(self, message: Mapping[str, JsonValue]) -> _TurnState | None:
+        transcript: Final = message.get("transcript")
+        if not isinstance(transcript, str):
+            raise MuseProtocolError("transcript event has invalid transcript")
+        if not transcript and message.get("turnId") is None and self._active_turn_id is None:
+            return None
+        turn: Final = self._turn(self._transcript_turn_id(message))
+        if message.get("final") is True:
+            self._finish(turn, transcript)
+        elif turn.final_text is None:
+            turn.latest_partial = transcript
+        return turn
+
+    def _speech_end(self, message: Mapping[str, JsonValue]) -> _TurnState:
+        turn: Final = self._turn(_required_turn_id(message, "speechEnd"))
+        turn.stopped = True
+        return turn
+
+    def _speech_complete(self, message: Mapping[str, JsonValue]) -> _TurnState:
+        transcript: Final = message.get("transcript")
+        if not isinstance(transcript, str):
+            raise MuseProtocolError("speechComplete event has invalid transcript")
+        turn: Final = self._turn(_required_turn_id(message, "speechComplete"))
+        self._finish(turn, transcript)
+        return turn
+
+    def _finish(self, turn: _TurnState, transcript: str) -> None:
+        turn.finish(transcript)
+        self._release_active(turn)
+
+    def _release_active(self, turn: _TurnState) -> None:
+        if self._active_turn_id == turn.item_id:
+            self._active_turn_id = None
+
+    def _update_audio_progress(self, message: Mapping[str, JsonValue]) -> None:
+        processed_ms: Final = message.get("audioProcessedMs")
+        if (
+            isinstance(processed_ms, bool)
+            or not isinstance(processed_ms, (int, float))
+            or not math.isfinite(processed_ms)
+            or processed_ms < 0
+        ):
+            raise MuseProtocolError("audioProgress event has invalid audioProcessedMs")
+        if processed_ms <= self._last_audio_processed_ms:
+            return
+        self._unbilled_seconds += (float(processed_ms) - self._last_audio_processed_ms) / 1000
+        self._last_audio_processed_ms = float(processed_ms)
+
+    def _transcript_turn_id(self, message: Mapping[str, JsonValue]) -> str:
+        if message.get("turnId") is not None:
+            return _required_turn_id(message, "transcript")
+        if self._active_turn_id is not None:
+            return self._active_turn_id
+        if self._mode != "PUSH_TO_TALK":
+            raise MuseProtocolError("transcript event is missing turnId outside an active turn")
+        turn_id: Final = f"item_{uuid.uuid4().hex}"
+        self._active_turn_id = turn_id
+        return turn_id
+
+
+class MetaRealtimeConfig(BaseRealtimeConfig):
+    def __init__(
+        self,
+        *,
+        monotonic: Callable[[], float] = time.monotonic,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    ) -> None:
+        self._monotonic: Final = monotonic
+        self._sleep: Final = sleep
+        self._transformer: Final = MuseEventTransformer()
+        self._access_token: str | None = None
+        self._config: MuseSessionConfig | None = None
+        self._pending_audio: bytes = b""
+        self._end_stream_sent: bool = False
+        self._pacing_origin: float | None = None
+        self._sent_duration: float = 0.0
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseRealtimeConfig contract
+        model: str,
+        api_key: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseRealtimeConfig contract
+        token: Final = api_key or get_secret_str("META_API_KEY")
+        if token is None:
+            raise ValueError("api_key is required for Meta API calls")
+        self._access_token = normalize_access_token(token)
+        return headers
+
+    def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str:
+        if _normalize_model(model) != MUSE_MODEL:
+            raise ValueError(f"Unsupported Meta realtime model: {model}")
+        return build_muse_realtime_url(api_base)
+
+    def is_setup_message(self, msg_obj: Mapping[str, object]) -> bool:
+        return "authorization" in msg_obj
+
+    def transform_session_created_event(
+        self,
+        model: str,
+        logging_session_id: str,
+        session_configuration_request: str | None = None,
+    ) -> OpenAIRealtimeTranscriptionSessionCreated:
+        return session_created_event(_DEFAULT_SESSION_CONFIG, logging_session_id)
+
+    def transform_realtime_request(
+        self,
+        message: str,
+        model: str,
+        session_configuration_request: str | None = None,
+    ) -> tuple[str | bytes, ...]:
+        request: Final = _json_object(message)
+        event_type: Final = request.get("type")
+        if event_type in ("session.update", "transcription_session.update"):
+            return self._configure(message, model)
+        if event_type == "input_audio_buffer.append":
+            return self._append_audio(request)
+        if event_type == "input_audio_buffer.commit":
+            return self._flush_audio(end_stream=self._require_config().mode == "PUSH_TO_TALK")
+        if event_type == "input_audio_buffer.end":
+            return self._flush_audio(end_stream=True)
+        if event_type == "input_audio_buffer.clear":
+            self._pending_audio = b""
+            return ()
+        verbose_logger.debug("Meta realtime: dropping unsupported client event %s", event_type)
+        return ()
+
+    async def pace_backend_send(self, message: bytes) -> None:
+        now: Final = self._monotonic()
+        origin: Final = self._pacing_origin
+        effective_origin: Final = (
+            now - self._sent_duration if origin is None or now > origin + self._sent_duration else origin
+        )
+        delay: Final = effective_origin + self._sent_duration - now
+        if delay > 0:
+            await self._sleep(delay)
+        self._pacing_origin = effective_origin
+        self._sent_duration += len(message) / self._require_config().bytes_per_second
+
+    def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
+        return self._transformer.take_unbilled_usage()
+
+    def transform_realtime_response(
+        self,
+        message: str | bytes,
+        model: str,
+        logging_obj: LiteLLMLoggingObj,
+        realtime_response_transform_input: RealtimeResponseTransformInput,
+    ) -> RealtimeResponseTypedDict:
+        payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message
+        result: Final[RealtimeResponseTypedDict] = {
+            "response": list(self._backend_events(payload)),  # mutable-ok: RealtimeResponseTypedDict.response is a list
+            "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"),
+            "current_response_id": realtime_response_transform_input.get("current_response_id"),
+            "current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"),
+            "current_conversation_id": realtime_response_transform_input.get("current_conversation_id"),
+            "current_item_chunks": realtime_response_transform_input.get("current_item_chunks"),
+            "current_delta_type": realtime_response_transform_input.get("current_delta_type"),
+            "session_configuration_request": realtime_response_transform_input.get("session_configuration_request"),
+        }
+        return result
+
+    def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]:
+        frame: Final = _json_object(payload)
+        session_id: Final = frame.get("sessionId")
+        if session_id is None:
+            return self._transformer.transform(frame)
+        if not isinstance(session_id, str) or not session_id.strip():
+            raise MuseProtocolError("provider returned an invalid handshake response")
+        return (session_created_event(self._require_config(), session_id.strip()),)
+
+    def _configure(self, message: str, model: str) -> tuple[str, ...]:
+        if self._config is not None:
+            verbose_logger.debug("Meta realtime: ignoring session.update after the Muse handshake was sent")
+            return ()
+        access_token: Final = self._access_token
+        if access_token is None:
+            raise MuseProtocolError("Meta API key was not validated before the session was configured")
+        config: Final = parse_session_update(message, model)
+        self._config = config
+        self._transformer.configure(config)
+        return (json.dumps(config.handshake(access_token), separators=(",", ":")),)
+
+    def _append_audio(self, request: Mapping[str, JsonValue]) -> tuple[bytes, ...]:
+        config: Final = self._require_config()
+        encoded: Final = request.get("audio")
+        if not isinstance(encoded, str):
+            raise MuseProtocolError("Audio must be a base64 string")
+        if len(encoded) > config.max_encoded_append_bytes:
+            raise MuseProtocolError("Audio append exceeds the four-second backlog limit")
+        try:
+            audio: Final = base64.b64decode(encoded, validate=True)
+        except (binascii.Error, ValueError):
+            raise MuseProtocolError("Audio must be valid base64") from None
+        if len(audio) % 2:
+            raise MuseProtocolError("PCM16 audio must contain complete samples")
+        buffered: Final = self._pending_audio + audio
+        packet_end: Final = len(buffered) - len(buffered) % config.packet_bytes
+        self._pending_audio = buffered[packet_end:]
+        return tuple(
+            buffered[start : start + config.packet_bytes] for start in range(0, packet_end, config.packet_bytes)
+        )
+
+    def _flush_audio(self, *, end_stream: bool) -> tuple[str | bytes, ...]:
+        remainder: Final = self._pending_audio
+        self._pending_audio = b""
+        frames: Final[tuple[bytes, ...]] = (remainder,) if remainder else ()
+        if not end_stream or self._end_stream_sent:
+            return frames
+        self._end_stream_sent = True
+        return (*frames, _END_STREAM)
+
+    def _require_config(self) -> MuseSessionConfig:
+        if self._config is None:
+            raise MuseProtocolError("session.update must configure the Muse session before audio is sent")
+        return self._config

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -42,6 +42,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     stream_item_field,
     stream_item_fingerprint,
     stream_item_items,
+    unappliable_request_rewrite,
 )
 from litellm.main import stream_chunk_builder
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@@ -196,6 +197,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             else:
                 # Step 3: Map guardrail responses back to original message structure
                 if guardrailed_texts and texts_to_check:
+                    if len(guardrailed_texts) != len(text_task_mappings):
+                        raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
                     await self._apply_guardrail_responses_to_input_texts(
                         messages=messages,
                         responses=guardrailed_texts,
@@ -210,6 +213,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                         task_mappings=tool_call_task_mappings,
                     )
 
+        elif (
+            not images_to_check
+            and not guardrail_to_apply.records_own_guardrail_information
+            and (not_run_reason := self._not_run_reason(messages)) is not None
+        ):
+            guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response=not_run_reason,
+                request_data=data,
+                guardrail_status="not_run",
+            )
+
         verbose_proxy_logger.debug(
             "OpenAI Chat Completions: Processed input messages: %s",
             data.get("messages"),
@@ -217,6 +231,28 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         return data
 
+    def _not_run_reason(
+        self,
+        messages: Sequence[dict[str, Any]],  # mutable-ok: raw request messages consumed by _extract_inputs
+    ) -> str | None:
+        """Why nothing was scanned, or None when the only unscoped content is images, which this handler never scans."""
+        texts: Final[list[str]] = []  # mutable-ok: filled by _extract_inputs
+        images: Final[list[str]] = []  # mutable-ok: filled by _extract_inputs
+        tool_calls: Final[list[ChatCompletionToolParam]] = []  # mutable-ok: filled by _extract_inputs
+        for msg_idx, message in enumerate(messages):
+            self._extract_inputs(
+                message=message,
+                msg_idx=msg_idx,
+                texts_to_check=texts,
+                images_to_check=images,
+                tool_calls_to_check=tool_calls,
+                text_task_mappings=[],  # mutable-ok: required by _extract_inputs, unused here
+                tool_call_task_mappings=[],  # mutable-ok: required by _extract_inputs, unused here
+            )
+        if texts or tool_calls:
+            return "no scannable content after message scoping"
+        return None if images else "no scannable content"
+
     def extract_request_tool_names(self, data: dict) -> list[str]:
         """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name)."""
         names: Final[list[str]] = []

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -56,6 +56,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     stream_item_field,
     stream_item_fingerprint,
     stream_item_items,
+    unappliable_request_rewrite,
 )
 from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
 from litellm.responses.litellm_completion_transformation.transformation import (
@@ -495,13 +496,13 @@ class OpenAIResponsesHandler(BaseTranslation):
                 data["instructions"] = written_back.instructions  # rebind-ok: data is an out-param
         elif isinstance(input_data, str):
             guardrailed_texts: Final = guardrailed_inputs.get("texts") or ()
+            if len(guardrailed_texts) > 1:
+                raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
             data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data  # rebind-ok: data is an out-param
         else:
             rewritten_texts: Final = guardrailed_inputs.get("texts") or ()
             if len(rewritten_texts) != len(extracted.task_mappings):
-                from litellm.proxy.policy_engine.pipeline_executor import UnappliableRequestRewrite
-
-                raise UnappliableRequestRewrite(guardrail_to_apply.guardrail_name or "unknown")
+                raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
             await self._apply_guardrail_responses_to_input(
                 messages=input_data,
                 responses=rewritten_texts,

litellm/llms/openai/responses/guardrail_translation/tool_merge.py

@@ -6,8 +6,10 @@ from typing import Final, TypeAlias
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
 from litellm.responses.litellm_completion_transformation.transformation import (
     NAMESPACE_DESCRIPTION_SEPARATOR,
+    NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS,
     LiteLLMCompletionResponsesConfig,
 )
 
@@ -34,8 +36,8 @@ def _validated_tools(values: Iterable[object]) -> tuple[Tool, ...]:
     return tuple(tool for tool in validated if tool is not None)
 
 
-def _is_function(tool: Tool) -> bool:
-    return tool.get("type") == "function"
+def _has_chat_tool(member: Tool) -> bool:
+    return member.get("type") in NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS
 
 
 def _chat_tool_key(tool: Tool) -> str:
@@ -67,18 +69,19 @@ def _function_fields(tool: Tool) -> Tool:
     return function if function is not None else MappingProxyType({})
 
 
-def _without_namespace_prefix(key: str, value: object, prefix: str) -> object:
-    if key != "description" or not isinstance(value, str) or not value.startswith(prefix):
+def _member_description(key: str, value: object, prefix: str, suffix: str) -> object:
+    if key != "description" or not isinstance(value, str):
         return value
-    return value[len(prefix) :]
+    return value.replace(prefix, "", 1).replace(suffix, "", 1)
 
 
 def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_description: str) -> Tool:
     flattened_function: Final = _function_fields(flattened)
     prefix: Final = f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}" if namespace_description else ""
+    suffix: Final = custom_tool_grammar_suffix(member.get("format")) if member.get("type") == "custom" else ""
     changed_function: Final = MappingProxyType(
         {
-            key: _without_namespace_prefix(key, value, prefix)
+            key: _member_description(key, value, prefix, suffix)
             for key, value in _function_fields(guardrailed).items()
             if flattened_function.get(key) != value
         }
@@ -93,8 +96,8 @@ def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_
     return {**member, **changed_extras, **changed_function}  # mutable-ok: json.dumps rejects MappingProxyType
 
 
-def _rebuilt_function_members(
-    function_members: Sequence[Tool],
+def _rebuilt_flattened_members(
+    flattened_members: Sequence[Tool],
     flattened_group: Sequence[Tool],
     group_keys: Sequence[IndexedKey],
     guardrailed_by_key: Mapping[IndexedKey, Tool],
@@ -106,7 +109,7 @@ def _rebuilt_function_members(
         else member
         if guardrailed_by_key[key] == flattened
         else _rebuilt_member(member, flattened, guardrailed_by_key[key], namespace_description)
-        for member, flattened, key in zip(function_members, flattened_group, group_keys)
+        for member, flattened, key in zip(flattened_members, flattened_group, group_keys)
     )
 
 
@@ -118,9 +121,9 @@ def _rebuilt_namespace(
     guardrailed_by_key: Mapping[IndexedKey, Tool],
 ) -> tuple[Tool, ...]:
     namespace_description: Final = str(original.get("description") or "")
-    rebuilt_functions: Final = iter(
-        _rebuilt_function_members(
-            tuple(member for member in members if _is_function(member)),
+    rebuilt_flattened: Final = iter(
+        _rebuilt_flattened_members(
+            tuple(member for member in members if _has_chat_tool(member)),
             flattened_group,
             group_keys,
             guardrailed_by_key,
@@ -129,7 +132,7 @@ def _rebuilt_namespace(
     )
     rebuilt_members: Final = tuple(
         rebuilt
-        for rebuilt in (next(rebuilt_functions) if _is_function(member) else member for member in members)
+        for rebuilt in (next(rebuilt_flattened) if _has_chat_tool(member) else member for member in members)
         if rebuilt is not None
     )
     if not rebuilt_members:
@@ -149,7 +152,7 @@ def _merged_original(
     if guardrailed_group == tuple(flattened_group):
         return (original,)
     members: Final = _namespace_members(original) if original.get("type") == "namespace" else ()
-    if members and sum(map(_is_function, members)) == len(flattened_group):
+    if members and sum(map(_has_chat_tool, members)) == len(flattened_group):
         return _rebuilt_namespace(original, members, flattened_group, group_keys, guardrailed_by_key)
     if not guardrailed_group:
         return ()

litellm/llms/openai_like/providers.json

@@ -173,7 +173,7 @@
     "api_key_env": "META_API_KEY",
     "api_base_env": "META_API_BASE",
     "base_class": "openai_gpt",
-    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages", "/v1/realtime"]
   },
   "cognition": {
     "base_url": "https://api.cognition.ai/v1",

litellm/llms/vertex_ai/batches/transformation.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final
 from urllib.parse import unquote
 
@@ -8,7 +9,36 @@ from litellm.llms.vertex_ai.common_utils import (
 )
 from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
 from litellm.types.llms.vertex_ai import *
-from litellm.types.utils import LiteLLMBatch
+from litellm.types.utils import LiteLLMBatch, PromptTokensDetailsWrapper
+
+
+def vertex_prompt_tokens_details(
+    usage_metadata: Mapping[str, object],
+) -> PromptTokensDetailsWrapper | None:
+    raw_details: Final = usage_metadata.get("promptTokensDetails")
+    if not isinstance(raw_details, list):
+        return None
+
+    def _normalize(detail: object) -> tuple[str, int] | None:
+        if not isinstance(detail, Mapping):
+            return None
+        modality: Final = detail.get("modality")
+        token_count: Final = detail.get("tokenCount")
+        if not isinstance(modality, str) or not isinstance(token_count, int):
+            return None
+        return modality.upper(), token_count
+
+    parsed_details: Final = tuple(_normalize(detail) for detail in raw_details)
+    normalized: Final = tuple(detail for detail in parsed_details if detail is not None)
+    if len(normalized) != len(parsed_details):
+        return None
+
+    return PromptTokensDetailsWrapper(
+        text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")),
+        audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"),
+        image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"),
+        video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"),
+    )
 
 
 class VertexAIBatchTransformation:

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py

@@ -298,8 +298,6 @@ def transform_openai_input_gemini_embed_content(
 
 
 _IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"})
-_VIDEO_TOKENS_PER_SECOND: Final = 258.0
-_AUDIO_TOKENS_PER_SECOND: Final = 32.0
 _usage_metadata_adapter: Final = TypeAdapter(UsageMetadata)
 
 
@@ -339,11 +337,12 @@ def _is_image_element(
     return False
 
 
-def _count_input_images(
+def _is_image_only_input(
     input: GeminiEmbeddingInput,
     resolved_files: Mapping[str, Mapping[str, str]],
-) -> int:
-    return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files))
+) -> bool:
+    elements: Final = _flatten_input(input)
+    return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements)
 
 
 def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int:
@@ -372,30 +371,29 @@ def _usage_from_embed_content_response(
     total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens
 
     details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
+    if not details:
+        return Usage(
+            prompt_tokens=prompt_tokens,
+            total_tokens=total_tokens,
+            prompt_tokens_details=PromptTokensDetailsWrapper(
+                text_tokens=0,
+                image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0,
+            ),
+        )
+
     text_tokens: Final = _tokens_for_modality(details, "TEXT")
     audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
+    image_tokens: Final = _tokens_for_modality(details, "IMAGE")
     video_tokens: Final = _tokens_for_modality(details, "VIDEO")
-    image_count: Final = _count_input_images(input, resolved_files)
-
-    video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
-    audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
-
-    # generic_cost_per_token rewrites text_tokens to the full prompt minus
-    # other modalities when both text_tokens and image_count are zero. For
-    # video, that misallocates video tokens to text; a 1-token floor sidesteps
-    # the rewrite and keeps billing on input_cost_per_video_per_second.
-    needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
-    resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
 
     return Usage(
         prompt_tokens=prompt_tokens,
         total_tokens=total_tokens,
         prompt_tokens_details=PromptTokensDetailsWrapper(
-            text_tokens=resolved_text_tokens,
+            text_tokens=text_tokens,
             audio_tokens=audio_tokens,
-            image_count=image_count,
-            video_length_seconds=video_length_seconds,
-            audio_length_seconds=audio_length_seconds,
+            image_tokens=image_tokens,
+            video_tokens=video_tokens,
         ),
     )
 
@@ -415,8 +413,7 @@ def process_embed_content_response(
         model_response: EmbeddingResponse to populate
         model: Model name
         response_json: Raw JSON response from embedContent endpoint
-        resolved_files: Mapping of file references (files/abc) to {mime_type, uri},
-            used to bill resolved image references at the per-image rate
+        resolved_files: Mapping of file references to resolved metadata
 
     Returns:
         EmbeddingResponse with single embedding

litellm/main.py

@@ -2592,7 +2592,9 @@ def _complete_custom_openai(
             copilot_headers.update(extra_headers)
         extra_headers = copilot_headers
 
-    if extra_headers is not None:
+    use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER")
+
+    if extra_headers is not None and not use_base_llm_http_handler:
         optional_params["extra_headers"] = extra_headers
 
     if litellm.enable_preview_features and metadata is not None:  # [PREVIEW] allow metadata to be passed to OPENAI
@@ -2609,8 +2611,6 @@ def _complete_custom_openai(
             optional_params[k] = v
 
     ## COMPLETION CALL
-    use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER")
-
     try:
         if use_base_llm_http_handler:
             response = base_llm_http_handler.completion(

litellm/ocr/legacy.py

@@ -28,6 +28,7 @@ from litellm.llms.base_llm.ocr.transformation import (
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.ocr.input import FileReader
 from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import CustomPricingLiteLLMParams
 from litellm.utils import ProviderConfigManager, client
 
 base_llm_http_handler: Final = BaseLLMHTTPHandler()
@@ -149,6 +150,7 @@ def _prepare_ocr_request(
         litellm_params={
             "litellm_call_id": litellm_call_id,
             "api_base": resolved_api_base,
+            **litellm_params.model_dump(include=frozenset(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
         },
         custom_llm_provider=custom_llm_provider,
     )

litellm/proxy/_experimental/mcp_server/AGENTS.md

@@ -67,9 +67,8 @@ module materially harder to understand.
   auth, SSE, streamable HTTP, and stdio as separate flows. Do not collapse them
   behind a single generic branch unless tests prove every mode still behaves
   correctly.
-- Be especially careful with `available_on_public_internet: false` combined with
-  `delegate_auth_to_upstream: true`. The local `CLAUDE.md` explains the anonymous
-  upstream PKCE path that must remain intentional.
+- Be especially careful with legacy `delegate_auth_to_upstream: true`. The local
+  `CLAUDE.md` explains its admitted replacement and public discovery contract.
 - Keep database-backed fields in sync across migrations, typed models under
   `litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this
   package, and dashboard state when the field is user-visible.

litellm/proxy/_experimental/mcp_server/CLAUDE.md

@@ -1 +1 @@
-MCP note: **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive - not `client_credentials`)** - LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database
+MCP note: **`auth_type: oauth2` with `delegate_auth_to_upstream: true` is deprecated** - LiteLLM admission is required for matching MCP routes. Use `auth_type: oauth_delegate` for client-forwarded OAuth. OAuth discovery endpoints stay public so clients can start the RFC 9728 flow

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -129,10 +129,9 @@ def _is_mcp_passthrough_cold_start(mcp_servers: list[str] | None, client_ip: str
     spec-compliant WWW-Authenticate challenge instead of surfacing a generic
     admission error.
 
-    Uses "all" semantics (mirrors
-    :meth:`MCPRequestHandler._target_servers_delegate_auth_to_upstream`): one
-    non-passthrough target in a co-targeted set must not flip the bypass open
-    for the others. Fails closed when any target cannot be resolved."""
+    Uses "all" semantics: one non-passthrough target in a co-targeted set must
+    not flip the bypass open for the others. Fails closed when any target
+    cannot be resolved."""
     if not mcp_servers:
         return False
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
@@ -146,6 +145,27 @@ def _is_mcp_passthrough_cold_start(mcp_servers: list[str] | None, client_ip: str
     return True
 
 
+def _is_legacy_delegate_cold_start(mcp_servers: list[str] | None, client_ip: str | None) -> bool:
+    """Allow only credential-free legacy delegates to reach the route's OAuth challenge."""
+    if not mcp_servers:
+        return False
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+        MCPServerManager,
+        global_mcp_server_manager,
+    )
+    from litellm.types.mcp import MCPAuth
+
+    for name in mcp_servers:
+        server = global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip)
+        if server is None or server.auth_type != MCPAuth.oauth2:
+            return False
+        if server.delegate_auth_to_upstream is not True:
+            return False
+        if MCPServerManager.effective_oauth2_flow(server) == "client_credentials":
+            return False
+    return True
+
+
 def _is_litellm_auth_admission_error(exc: Exception) -> bool:
     if isinstance(exc, HTTPException):
         return exc.status_code == 401
@@ -277,9 +297,18 @@ def _admission_failure_fallback(
         mcp_servers_from_path is not None
         and not _has_client_supplied_mcp_auth(mcp_auth_header, mcp_server_auth_headers)
         and _is_litellm_auth_admission_error(exc)
-        and _is_mcp_passthrough_cold_start(
-            mcp_servers_from_path,
-            client_ip=IPAddressUtils.get_mcp_client_ip(request),
+        and (
+            _is_mcp_passthrough_cold_start(
+                mcp_servers_from_path,
+                client_ip=IPAddressUtils.get_mcp_client_ip(request),
+            )
+            or (
+                not bearer_presented
+                and _is_legacy_delegate_cold_start(
+                    mcp_servers_from_path,
+                    client_ip=IPAddressUtils.get_mcp_client_ip(request),
+                )
+            )
         )
     ):
         verbose_logger.debug("MCP pass-through cold start: deferring admission to route 401 emitter")
@@ -434,22 +463,6 @@ class MCPRequestHandler:
                 api_key=f"Bearer {_get_bearer_token_or_received_api_key(litellm_api_key)}",
                 request=request,
             )
-        elif MCPRequestHandler._target_servers_delegate_auth_to_upstream(
-            path=request_route,
-            mcp_servers=mcp_servers,
-            client_ip=IPAddressUtils.get_mcp_client_ip(request),
-        ):
-            # Operator opted this oauth2 server into upstream-delegated auth: the
-            # client authenticates directly with the upstream MCP server, so any
-            # Authorization bearer is an upstream token, never a LiteLLM key. Skip
-            # LiteLLM validation entirely — covering both the no-credential
-            # discovery request and the authenticated call carrying the upstream
-            # bearer — so a tool call that succeeds never carries a phantom 401
-            # auth span; the bearer is forwarded upstream unchanged. Gated by
-            # _target_servers_delegate_auth_to_upstream, which returns True only
-            # when EVERY target is auth_type=oauth2 with delegate_auth_to_upstream
-            # set; fails closed otherwise.
-            validated_user_api_key_auth = UserAPIKeyAuth()
         elif MCPRequestHandler._target_servers_are_true_passthrough(
             path=request_route,
             mcp_servers=mcp_servers,
@@ -660,64 +673,6 @@ class MCPRequestHandler:
             return [single_server_match.group(1)]
         return [servers_and_path]
 
-    @staticmethod
-    def _target_servers_delegate_auth_to_upstream(
-        path: str, mcp_servers: list[str] | None, client_ip: str | None
-    ) -> bool:
-        """
-        True only when EVERY MCP server the request targets is configured for
-        ``auth_type == oauth2`` AND has ``delegate_auth_to_upstream=True``.
-        Fails closed when any target does not opt in or cannot be resolved.
-
-        Used by :meth:`process_mcp_request` to skip LiteLLM API-key/SSO auth
-        entirely (PKCE passthrough) so the client authenticates directly with
-        the upstream MCP server. Mixed-target requests (e.g. one delegated +
-        one non-delegated server) fall back to normal LiteLLM auth.
-        """
-        # Inline imports avoid a circular dependency: mcp_server_manager imports
-        # from this module.
-        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-            MCPServerManager,
-            global_mcp_server_manager,
-        )
-        from litellm.types.mcp import MCPAuth
-
-        # Must mirror the downstream header-vs-path override
-        # (``extract_mcp_auth_context``) or an attacker could set
-        # ``x-mcp-servers`` to a delegate-enabled server while the URL path
-        # targets a non-delegate server, skipping LiteLLM auth for it.
-        target_names: Final = MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers)
-        if not target_names:
-            return False
-
-        for name in target_names:
-            server = global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip)
-            if server is None or server.auth_type != MCPAuth.oauth2:
-                return False
-            # `is True` is intentional: opt-in must be an explicit boolean
-            # True. A MagicMock attribute (in tests) or any other truthy
-            # non-bool must not silently enable the bypass.
-            if getattr(server, "delegate_auth_to_upstream", False) is not True:
-                return False
-            # Never delegate for M2M (client_credentials) servers: LiteLLM
-            # fetches the upstream token automatically using stored credentials,
-            # so allowing anonymous bypass would let any external caller invoke
-            # tools authenticated as LiteLLM's service account.
-            #
-            # Resolve the flow rather than reading has_client_credentials directly:
-            # this is a security gate, and a legacy row whose oauth2_flow was never
-            # stamped still carries the M2M credential shape (client_id/secret +
-            # token_url, no authorization_url). Treating an unstamped-but-M2M-shaped
-            # row as non-M2M here would reopen the anonymous bypass the explicit
-            # column no longer closes on its own. Shares the one resolution helper
-            # with the egress backstop and the anonymous-delegate allowlist; all fail
-            # closed on the ambiguous shape and are removed together once no null rows
-            # remain. A pure-PKCE delegate server (no stored credentials) resolves to a
-            # non-M2M flow and keeps its bypass.
-            if MCPServerManager.effective_oauth2_flow(server) == "client_credentials":
-                return False
-        return True
-
     @staticmethod
     def _target_servers_are_true_passthrough(path: str, mcp_servers: list[str] | None, client_ip: str | None) -> bool:
         """
@@ -726,7 +681,7 @@ class MCPRequestHandler:
 
         Used by :meth:`process_mcp_request` to skip LiteLLM admission auth entirely: the gateway is a
         transparent proxy and the caller's ``Authorization`` is an upstream token, never a LiteLLM key.
-        Mirrors :meth:`_target_servers_delegate_auth_to_upstream`; a mixed-target request keeps normal auth.
+        A mixed-target request keeps normal auth.
         """
         from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
             global_mcp_server_manager,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -1055,7 +1055,7 @@ def _should_strip_caller_authorization(
       pass-through cold-start case (RFC 9728) the bearer in
       ``Authorization`` is the upstream OAuth token and must be
       forwarded, so we keep it.
-    - **oauth_delegate servers**: admission always runs and there is no
+    - **Delegated OAuth servers**: admission always runs and there is no
       anonymous path, so the caller's separate ``Authorization`` is
       forwarded only when a distinct ``x-litellm-api-key`` carried
       admission. Without that header the ``Authorization`` *was* the
@@ -1075,12 +1075,17 @@ def _should_strip_caller_authorization(
         # upstream — it would override another user's stored credential. Delegate and
         # pass-through return None from to_server_spec and keep forwarding the bearer.
         return True
-    if not (mcp_server.is_oauth_passthrough or mcp_server.is_oauth_delegate):
+    is_delegated_oauth: Final = mcp_server.is_oauth_delegate or (
+        mcp_server.auth_type == MCPAuth.oauth2 and mcp_server.delegate_auth_to_upstream
+    )
+    if not (mcp_server.is_oauth_passthrough or is_delegated_oauth):
         return False
 
     has_explicit_litellm_admission_header: Final = _has_explicit_litellm_admission_header(raw_headers)
-    if mcp_server.is_oauth_delegate:
-        return not has_explicit_litellm_admission_header
+    if is_delegated_oauth:
+        return not has_explicit_litellm_admission_header or _authorization_is_litellm_admission_credential(
+            raw_headers, user_api_key_auth
+        )
     return _authorization_is_litellm_admission_credential(raw_headers, user_api_key_auth) or (
         user_api_key_auth is None and not has_explicit_litellm_admission_header
     )
@@ -1107,15 +1112,11 @@ def _authorization_is_litellm_admission_credential(
     That is the case when no usable ``x-litellm-api-key`` was sent, or when the client repeated the
     same key in both headers.
     """
-    if user_api_key_auth is None or not user_api_key_auth.api_key:
-        return False
     admission_header: Final = _raw_header_value(raw_headers, "x-litellm-api-key")
-    if not admission_header:
-        return True
     authorization: Final = _raw_header_value(raw_headers, "authorization")
-    return authorization is not None and strip_auth_scheme(authorization, "Bearer") == strip_auth_scheme(
-        admission_header, "Bearer"
-    )
+    if admission_header and authorization:
+        return strip_auth_scheme(authorization, "Bearer") == strip_auth_scheme(admission_header, "Bearer")
+    return bool(user_api_key_auth and user_api_key_auth.api_key and not admission_header)
 
 
 def _format_byok_openapi_auth_header(mcp_server: MCPServer, mcp_auth_header: str) -> str:
@@ -1453,22 +1454,19 @@ def _warn_on_server_name_fields(
     _warn("server_name", server_name)
 
 
-def _warn_internal_delegate_pkce_if_applicable(server: MCPServer, *, source: str) -> None:
-    """Surface internal + upstream PKCE delegate in logs for operators."""
+def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None:
+    """Direct legacy delegated OAuth configurations to the admitted replacement."""
     if server.auth_type != MCPAuth.oauth2:
         return
     if getattr(server, "delegate_auth_to_upstream", False) is not True:
         return
-    if getattr(server, "available_on_public_internet", True):
-        return
     if server.has_client_credentials:
         return
     label: Final = get_server_prefix(server)
     verbose_logger.warning(
-        "MCP server %r (id=%s, source=%s): internal-only (available_on_public_internet=false) "
-        "with delegate_auth_to_upstream=true. Anonymous callers can reach the upstream OAuth2 "
-        "/authorize flow and complete PKCE without a LiteLLM API key session; ensure the "
-        "upstream IdP and network enforce your access policy.",
+        "MCP server %r (id=%s, source=%s) uses deprecated auth_type=oauth2 with "
+        "delegate_auth_to_upstream=true. LiteLLM admission is now required; migrate to "
+        "auth_type=oauth_delegate for client-forwarded OAuth.",
         label,
         server.server_id,
         source,
@@ -2640,7 +2638,7 @@ class MCPServerManager:
                 oauth_identity_binding=server_config.get("oauth_identity_binding", None),
             )
             self._assign_unique_short_prefix(new_server)
-            _warn_internal_delegate_pkce_if_applicable(new_server, source="config")
+            _warn_legacy_delegate_auth_if_applicable(new_server, source="config")
             _warn_config_id_jag_server_outruns_sso(new_server)
             self._invalidate_discovery_lists(server_id)
             self.config_mcp_servers[server_id] = new_server
@@ -3185,7 +3183,7 @@ class MCPServerManager:
             timeout=getattr(mcp_server, "timeout", None),
             max_concurrent_requests=getattr(mcp_server, "max_concurrent_requests", None),
         )
-        _warn_internal_delegate_pkce_if_applicable(new_server, source="database")
+        _warn_legacy_delegate_auth_if_applicable(new_server, source="database")
         self._set_oauth_discovery_deferred(
             new_server.server_id,
             _requires_oauth_discovery(server_url, use_issuer_anchor, new_server),
@@ -3479,10 +3477,6 @@ class MCPServerManager:
                 )
             )
 
-            # For anonymous callers (no user_id, no role), also surface any
-            # servers the operator has opted into upstream-delegated auth.
-            # These servers handle their own auth at the upstream level, so
-            # LiteLLM granting access here does not bypass any security gate.
             is_anonymous: Final = not (
                 user_api_key_auth
                 and (
@@ -3492,23 +3486,12 @@ class MCPServerManager:
                 )
             )
             if is_anonymous:
-                delegate_server_ids: Final = [
+                passthrough_server_ids: Final = [
                     server.server_id
                     for server in self.get_registry().values()
-                    if (
-                        getattr(server, "auth_type", None) == MCPAuth.oauth2
-                        and getattr(server, "delegate_auth_to_upstream", False) is True
-                        # M2M servers must not be exposed anonymously: an
-                        # unauthenticated caller would get LiteLLM to proxy tool
-                        # calls using its stored client_credentials. Resolve the flow
-                        # rather than reading has_client_credentials so an unstamped
-                        # M2M-shape row (null column, verbatim-read as non-M2M) still
-                        # fails closed here, matching the anonymous-delegate auth gate.
-                        and MCPServerManager.effective_oauth2_flow(server) != "client_credentials"
-                    )
-                    or getattr(server, "auth_type", None) == MCPAuth.true_passthrough
+                    if getattr(server, "auth_type", None) == MCPAuth.true_passthrough
                 ]
-                combined_servers.update(delegate_server_ids)
+                combined_servers.update(passthrough_server_ids)
 
             restrict_allow_all: Final = (
                 resolved_general_settings.get("mcp_allow_all_keys_respects_mcp_scope", False)

litellm/proxy/_experimental/mcp_server/server.py

@@ -4257,20 +4257,6 @@ if MCP_AVAILABLE:
             return None
         return _get_authorization_header_from_scope(scope)
 
-    def _is_delegate_upstream_probe_target(server: MCPServer) -> bool:
-        """Whether ``server`` is an interactive delegate-auth server whose client-supplied
-        token should be preflighted upstream.
-
-        Mirrors the anonymous-delegate gate in ``get_allowed_mcp_servers``: the flow is
-        resolved via ``effective_oauth2_flow`` so an unstamped M2M-shape row fails closed
-        (its stored client credentials drive egress; the caller's bearer is irrelevant).
-        """
-        return (
-            server.auth_type == MCPAuth.oauth2
-            and server.delegate_auth_to_upstream is True
-            and MCPServerManager.effective_oauth2_flow(server) != "client_credentials"
-        )
-
     async def _probe_upstream_auth(
         url: str,
         auth_header: str,
@@ -4331,7 +4317,7 @@ if MCP_AVAILABLE:
         mcp_servers: list[str] | None,
         client_ip: str | None,
     ) -> None:
-        """Probe pass-through and delegate-auth upstream servers in parallel before the MCP session starts.
+        """Probe pass-through upstream servers in parallel before the MCP session starts.
 
         Only servers the caller's key is already authorized to reach are probed —
         the list is derived from _get_allowed_mcp_servers so that a user cannot
@@ -4343,38 +4329,9 @@ if MCP_AVAILABLE:
         if the upstream accepts it but forbids the caller.
         Fails-open: network errors are logged and the request is allowed through.
 
-        Delegate-auth servers (``auth_type=oauth2`` + ``delegate_auth_to_upstream``)
-        are probed with the caller's bare ``Authorization`` bearer. That bearer is only
-        an upstream token (never a LiteLLM key) when admission took the delegate bypass,
-        so the delegate target is resolved through ``get_mcp_server_by_name`` -- the same
-        resolver admission used -- rather than the wider allowed-server prefix/access-group
-        matching. A name that only reaches a delegate server via server_id or an access
-        group would have been admitted as a real LiteLLM key, so probing it would leak that
-        key upstream; requiring the admission-resolver match closes that gap. Without the
-        probe a rejected token is absorbed by the tools/list handler and masked as an empty
-        tool list. Gated to single-server routes so one rejected token cannot 401 a
-        multi-server aggregate connect, matching the OBO preflight gating; the challenge
-        echoes the requested name so aliased routes get the same resource_metadata URL as
-        the tokenless preemptive challenge.
         """
         forwarded_auth: Final = _get_forwarded_auth_from_scope(scope)
-        requested_single_target: Final = mcp_servers[0] if mcp_servers is not None and len(mcp_servers) == 1 else None
-        # The bare Authorization header (no x-litellm-api-key) is a valid upstream token
-        # only when admission classified it as one, i.e. the single requested name resolves
-        # to a delegate server under admission's own resolver. Resolve it the same way here
-        # so a server_id- or access-group-named delegate (which admission would have treated
-        # as a LiteLLM key) is never probed with that key.
-        delegate_server: Final = (
-            global_mcp_server_manager.get_mcp_server_by_name(requested_single_target, client_ip=client_ip)
-            if requested_single_target
-            else None
-        )
-        delegate_auth: Final = (
-            _get_authorization_header_from_scope(scope)
-            if delegate_server is not None and _is_delegate_upstream_probe_target(delegate_server)
-            else None
-        )
-        if not forwarded_auth and not delegate_auth:
+        if not forwarded_auth:
             return
 
         # Use the authorized server set, not the raw user-supplied names, so that
@@ -4384,35 +4341,20 @@ if MCP_AVAILABLE:
             mcp_servers=mcp_servers,
             client_ip=client_ip,
         )
-        passthrough_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = (
-            tuple(
-                (srv, forwarded_auth, srv.name)
-                for srv in allowed_servers
-                # Restrict to genuine OAuth pass-through servers (auth_type none +
-                # Authorization in extra_headers). Gateway-managed OAuth2 servers
-                # must not receive the ``resource_metadata=`` challenge emitted
-                # below — they require ``authorization_uri=`` pointing at the
-                # gateway AS metadata. ``is_oauth_passthrough`` already requires
-                # ``auth_type in (None, MCPAuth.none)``, which is mutually
-                # exclusive with ``has_client_credentials`` (oauth2 + M2M flow),
-                # so M2M servers are implicitly excluded here.
-                if srv.is_oauth_passthrough
-            )
-            if forwarded_auth
-            else ()
+        passthrough_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = tuple(
+            (srv, forwarded_auth, srv.name)
+            for srv in allowed_servers
+            # Restrict to genuine OAuth pass-through servers (auth_type none +
+            # Authorization in extra_headers). Gateway-managed OAuth2 servers
+            # must not receive the ``resource_metadata=`` challenge emitted
+            # below — they require ``authorization_uri=`` pointing at the
+            # gateway AS metadata. ``is_oauth_passthrough`` already requires
+            # ``auth_type in (None, MCPAuth.none)``, which is mutually
+            # exclusive with ``has_client_credentials`` (oauth2 + M2M flow),
+            # so M2M servers are implicitly excluded here.
+            if srv.is_oauth_passthrough
         )
-        # Probe the admission-resolved delegate server only when the caller is actually
-        # authorized for it (present in the IP-filtered allowed set), keyed by server_id.
-        delegate_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = (
-            tuple(
-                (srv, delegate_auth, requested_single_target)
-                for srv in allowed_servers
-                if delegate_server is not None and srv.server_id == delegate_server.server_id
-            )
-            if delegate_auth and requested_single_target
-            else ()
-        )
-        probe_targets: Final = passthrough_targets + delegate_targets
+        probe_targets: Final = passthrough_targets
         if not probe_targets:
             return
 

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,35 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
+2:I[347257,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"ClientPageRoot"]
+3:I[871135,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js","/litellm-asset-prefix/_next/static/chunks/1kabwy23ggmhn.js","/litellm-asset-prefix/_next/static/chunks/03ljmgnmrvuxw.js","/litellm-asset-prefix/_next/static/chunks/1fl3r3enx76vk.js","/litellm-asset-prefix/_next/static/chunks/2wi6ubg_xifzg.js","/litellm-asset-prefix/_next/static/chunks/14hu41xvpzmuj.js","/litellm-asset-prefix/_next/static/chunks/0eh6yvm8qswse.js","/litellm-asset-prefix/_next/static/chunks/0uo3fyy_3adzw.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1e5rsi2izekus.js","/litellm-asset-prefix/_next/static/chunks/1mxx3pzc7v4_x.js","/litellm-asset-prefix/_next/static/chunks/1fcix1vz8h1c8.js","/litellm-asset-prefix/_next/static/chunks/05vpfvve3-xds.js","/litellm-asset-prefix/_next/static/chunks/02erlqrgtvdvz.js","/litellm-asset-prefix/_next/static/chunks/3fj8wylwf-stx.js","/litellm-asset-prefix/_next/static/chunks/27u46a0m025he.js","/litellm-asset-prefix/_next/static/chunks/3c013ns4vt0zs.js","/litellm-asset-prefix/_next/static/chunks/1gvvrnrpw-7_u.js","/litellm-asset-prefix/_next/static/chunks/0fg9nx_731nkm.js","/litellm-asset-prefix/_next/static/chunks/1t7m5lyrljbza.js","/litellm-asset-prefix/_next/static/chunks/371aylk03p56q.js","/litellm-asset-prefix/_next/static/chunks/0n3xjld_n2chp.js","/litellm-asset-prefix/_next/static/chunks/2oyhu8rllo9v-.js","/litellm-asset-prefix/_next/static/chunks/3mwt8ofux_ic-.js","/litellm-asset-prefix/_next/static/chunks/0d17ojhl52r4k.js","/litellm-asset-prefix/_next/static/chunks/0xl3regan_n7s.js","/litellm-asset-prefix/_next/static/chunks/0ixfd4seits4-.js"],"default"]
+6:I[897367,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
+b:I[897367,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"ViewportBoundary"]
+c:I[897367,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"MetadataBoundary"]
+d:I[27201,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"IconMark"]
+f:I[92825,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"ClientSegmentRoot"]
+10:I[216370,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js","/litellm-asset-prefix/_next/static/chunks/1kabwy23ggmhn.js","/litellm-asset-prefix/_next/static/chunks/03ljmgnmrvuxw.js","/litellm-asset-prefix/_next/static/chunks/1fl3r3enx76vk.js","/litellm-asset-prefix/_next/static/chunks/2wi6ubg_xifzg.js","/litellm-asset-prefix/_next/static/chunks/14hu41xvpzmuj.js","/litellm-asset-prefix/_next/static/chunks/0eh6yvm8qswse.js","/litellm-asset-prefix/_next/static/chunks/0uo3fyy_3adzw.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1e5rsi2izekus.js"],"default"]
+11:I[339756,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"default"]
+12:I[837457,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"default"]
+a:X
+0:{"buildId":"N8M8GUEWcUrwZCaluei8R","data":[{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1mxx3pzc7v4_x.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1fcix1vz8h1c8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/05vpfvve3-xds.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/02erlqrgtvdvz.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3fj8wylwf-stx.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/27u46a0m025he.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/3c013ns4vt0zs.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1gvvrnrpw-7_u.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0fg9nx_731nkm.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/1t7m5lyrljbza.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/371aylk03p56q.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0n3xjld_n2chp.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2oyhu8rllo9v-.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/3mwt8ofux_ic-.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0d17ojhl52r4k.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0xl3regan_n7s.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0ixfd4seits4-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":"$@9","staleTime":"$a","varyParams":null},{"rsc":["$","$1","h",{"children":[null,["$","$Lb",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$7",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$Ld","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":"$@e","staleTime":"$a","varyParams":null},{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1kabwy23ggmhn.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/03ljmgnmrvuxw.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1fl3r3enx76vk.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2wi6ubg_xifzg.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/14hu41xvpzmuj.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0eh6yvm8qswse.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0uo3fyy_3adzw.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1e5rsi2izekus.js","async":true}]],["$","$Lf",null,{"Component":"$10","slots":{"children":["$","$L11",null,{"parallelRouterKey":"children","template":["$","$L12",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":"$L13"}]],[]]}]},"serverProvidedParams":{"params":"$0:data:0:rsc:props:children:0:props:serverProvidedParams:params","promises":["$@14"]}}]]}],"isPartial":"$@15","staleTime":"$a","varyParams":null},{"rsc":"$L16","isPartial":"$@17","staleTime":"$a","varyParams":null}],"isUpgradeableISRFallback":false,"a":"$@18","rootVaryParams":null,"needsRuntimeRequest":"$@19"}
+1a:I[363178,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"ThemeProvider"]
+1b:I[12985,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"NuqsAdapter"]
+1c:I[867271,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"default"]
+1d:I[557951,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"AuthProvider"]
+1e:I[713354,["/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js"],"Toaster"]
+:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/3rynlyl14avb-.css","style"]
 4:{}
-5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
+5:"$0:data:0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null
+13:["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]
+14:"$0:data:0:rsc:props:children:0:props:serverProvidedParams:params"
+16:["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3rynlyl14avb-.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/33t46atd3n2zd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3doe-1fpykdw3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1m-a1t8oh1ed0.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L1a",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L1b",null,{"children":["$","$L1c",null,{"children":[["$","$L1d",null,{"children":["$","$L11",null,{"parallelRouterKey":"children","template":["$","$L12",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:data:2:rsc:props:children:1:props:slots:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$13:props:children:1:props:style","children":404}],["$","div",null,{"style":"$13:props:children:2:props:style","children":["$","h2",null,{"style":"$13:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L1e",null,{}]]}]}]}]}]}]]}]
+a:300
+19:true
+a:C
+18:0
+e:"$undefined"
+17:"$undefined"
+9:"$undefined"
+15:"$undefined"

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +0,0 @@
-1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
-6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +0,0 @@
-1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
-4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
-0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,11 +0,0 @@
-1:"$Sreact.fragment"
-2:I[363178,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
-3:I[12985,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
-4:I[867271,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-5:I[557951,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
-6:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-7:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
-8:I[713354,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
-:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/3rynlyl14avb-.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"912qRXFjlEYHK3EAPXXTc"}
+0:{"tree":{"name":"","param":null,"prefetchHints":4176,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":4192,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":4256,"slots":null}}}}},"staleTime":300,"buildId":"N8M8GUEWcUrwZCaluei8R"}