chore: merge litellm_internal_staging into litellm_lit_7039_least_busy_shared_counts

.github/e2e-stack/assert_tests_ran.py

@@ -0,0 +1,38 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    selected: Final = tuple(sys.argv[2:])
+    try:
+        report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    except (ET.ParseError, OSError):
+        _ = sys.stdout.write("::error::could not read the test execution report\n")
+        return 1
+    cases: Final = tuple(report.iter("testcase"))
+    passed: Final = frozenset(
+        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+    )
+    missing: Final = tuple(path for path in selected if path not in passed)
+    for path in selected:
+        collected: Final = sum(case.get("file") == path for case in cases)
+        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
+        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
+    if (
+        selected
+        and not missing
+        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
+    ):
+        return 0
+    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/down.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+
+for pid_file in "${STACK_DIR}"/pids/*.pid; do
+  [[ -f "${pid_file}" ]] || continue
+  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
+  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
+  rm -f "${pid_file}"
+done
+
+for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+  docker rm -f "${container}" >/dev/null 2>&1
+done
+
+exit 0

.github/e2e-stack/secrets_to_env.py

@@ -0,0 +1,49 @@
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
+ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+MIN_MASKED_LENGTH: Final = 8
+
+
+def main() -> int:
+    env_path: Final = Path(sys.argv[1])
+    try:
+        secrets: Final = {
+            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
+        }
+    except (ValidationError, UnicodeError):
+        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
+        return 1
+    unusable: Final = tuple(
+        key
+        for key, value in secrets.items()
+        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
+    )
+    if unusable:
+        _ = sys.stderr.write(
+            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
+        )
+        return 1
+    for value in secrets.values():
+        if len(value) >= MIN_MASKED_LENGTH:
+            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
+    sys.stdout.flush()
+    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    try:
+        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            _ = handle.write("\n".join(lines) + "\n")
+    except OSError:
+        _ = sys.stderr.write("could not write the environment file\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/select_tests.py

@@ -0,0 +1,44 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+UNSUPPORTED: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/"
+    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
+    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+UNEXPANDED: Final = re.compile(r"[*?\[]")
+
+
+def is_selectable(path: str) -> bool:
+    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if is_selectable(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    canary: Final = tuple(sys.argv[1:])
+    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
+    if unexpanded:
+        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
+        return 1
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/up.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+CERTS_DIR="${STACK_DIR}/certs"
+LOGS_DIR="${STACK_DIR}/logs"
+PIDS_DIR="${STACK_DIR}/pids"
+
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
+JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
+
+LB_PORT="${E2E_LB_PORT:-4000}"
+GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
+GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
+BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
+REDIS_PORT="${E2E_REDIS_PORT:-6379}"
+DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
+DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
+DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
+DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
+DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
+JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
+JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+
+MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
+
+mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 755 "${CERTS_DIR}"
+
+log() { printf 'e2e-stack: %s\n' "$*"; }
+
+port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
+
+wait_for() {
+  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
+  until eval "${check}"; do
+    if ((SECONDS >= deadline)); then
+      log "timed out waiting for ${label}"
+      exit 1
+    fi
+    sleep 2
+  done
+  log "${label} is up"
+}
+
+if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
+  set -a
+  source "${REPO_ROOT}/tests/e2e/.env"
+  set +a
+fi
+
+if [[ -z "${DD_API_KEY:-}" ]]; then
+  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
+  exit 1
+fi
+export DD_SITE="${DD_SITE:-datadoghq.com}"
+
+if ! port_open "${DATABASE_PORT}"; then
+  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
+    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
+    "${POSTGRES_IMAGE}" >/dev/null
+fi
+wait_for "postgres" "port_open ${DATABASE_PORT}"
+
+if ! port_open "${JAEGER_QUERY_PORT}"; then
+  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
+    "${JAEGER_IMAGE}" >/dev/null
+fi
+wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
+
+openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
+openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
+  -subj "/CN=litellm-e2e-ca" \
+  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
+  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
+openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
+openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
+openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
+  -CAcreateserial -days 7 -sha256 \
+  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
+  -out "${CERTS_DIR}/server.crt" 2>/dev/null
+chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
+
+CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
+cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
+
+docker rm -f e2e-valkey >/dev/null 2>&1 || true
+docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
+  "${VALKEY_IMAGE}" valkey-server \
+  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
+  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
+  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
+VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
+wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
+${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
+wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
+
+CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
+CONFIG_PATH="${CONFIG_SOURCE}"
+if [[ "${REDIS_PORT}" != "6379" ]]; then
+  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
+  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
+fi
+
+SERVER_ENV=(
+  "LITELLM_MASTER_KEY=${MASTER_KEY}"
+  "DATABASE_HOST=${DATABASE_HOST}"
+  "DATABASE_PORT=${DATABASE_PORT}"
+  "DATABASE_USER=${DATABASE_USER}"
+  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
+  "DATABASE_NAME=${DATABASE_NAME}"
+  "DISABLE_SCHEMA_UPDATE=true"
+  "REDIS_HOST=127.0.0.1"
+  "REDIS_PORT=${REDIS_PORT}"
+  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
+  "CONFIG_FILE_PATH=${CONFIG_PATH}"
+  "STORE_MODEL_IN_DB=True"
+  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
+  "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
+  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
+  "PYTHONPATH=${REPO_ROOT}"
+)
+if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
+  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
+  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
+fi
+
+cd "${REPO_ROOT}"
+
+log "running migrations"
+env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
+
+start_server() {
+  local name="$1"; shift
+  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  echo $! > "${PIDS_DIR}/${name}.pid"
+}
+
+start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
+start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
+start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
+
+if [[ "$(uname)" == "Linux" ]]; then
+  NGINX_UPSTREAM_HOST=127.0.0.1
+  NGINX_DOCKER_ARGS=(--network host)
+else
+  NGINX_UPSTREAM_HOST=host.docker.internal
+  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
+fi
+
+cat > "${STACK_DIR}/nginx.conf" <<EOF
+events {}
+http {
+  map \$http_upgrade \$connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+  upstream litellm_gateways {
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
+  }
+  server {
+    listen ${LB_PORT};
+    client_max_body_size 100m;
+    location / {
+      proxy_pass http://litellm_gateways;
+      proxy_http_version 1.1;
+      proxy_set_header Host \$host;
+      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto \$scheme;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection \$connection_upgrade;
+      proxy_buffering off;
+      proxy_read_timeout 600s;
+      proxy_send_timeout 600s;
+    }
+  }
+}
+EOF
+
+docker rm -f e2e-nginx >/dev/null 2>&1 || true
+docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
+  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
+
+wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
+wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
+wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
+wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
+
+cat > "${STACK_DIR}/stack.env" <<EOF
+LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
+LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
+LITELLM_MASTER_KEY=${MASTER_KEY}
+REDIS_HOST=127.0.0.1
+REDIS_PORT=${REDIS_PORT}
+E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
+DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
+EOF
+
+log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

.github/workflows/_test-unit-base.yml

@@ -55,17 +55,14 @@ on:
 permissions:
   contents: read
 
+env:
+  UV_PYTHON: "3.12"
+
 jobs:
   run:
-    name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }}
+    name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.job-timeout-minutes }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-    env:
-      UV_PYTHON: ${{ matrix.python-version }}
     permissions:
       contents: read
       pull-requests: read
@@ -88,7 +85,7 @@ jobs:
         timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.UV_PYTHON }}
 
       - name: Set up uv
         if: steps.changes.outputs.decision != 'skip'
@@ -103,9 +100,9 @@ jobs:
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
+          key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-
+            ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
 
       - name: Cache the Rust build
         if: steps.changes.outputs.decision != 'skip'
@@ -139,7 +136,7 @@ jobs:
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
           DIST: ${{ inputs.dist }}
-          COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }}
+          COVERAGE_CORE: sysmon
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} \
@@ -166,7 +163,7 @@ jobs:
           fi
 
       - name: Save coverage report
-        if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip'
+        if: always() && steps.changes.outputs.decision != 'skip'
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/test-code-quality.yml

@@ -74,6 +74,9 @@ jobs:
       - name: check_workflow_startup_safety
         run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
 
+      - name: test_e2e_changed_gate
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-e2e-changed.yml

@@ -0,0 +1,240 @@
+name: e2e-changed-tests
+
+on:
+  pull_request:
+
+concurrency:
+  group: e2e-changed-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  detect:
+    name: Detect changed e2e tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      tests: ${{ steps.changed.outputs.tests }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: List the e2e test files this PR added or modified
+        id: changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
+            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
+            | grep -Fxq "${HEAD_SHA}"
+          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
+            --jq '.[] | select(.status != "removed") | .filename')"
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
+          tests="$(printf '%s\n' "${files}" \
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
+          if [ -n "${tests}" ]; then
+            echo "any=true" >> "${GITHUB_OUTPUT}"
+            echo "selected e2e tests: ${tests}"
+          else
+            echo "any=false" >> "${GITHUB_OUTPUT}"
+            echo "no changed e2e test files supported by this stack; nothing to run"
+          fi
+
+  run:
+    name: Run changed e2e tests against the stage-mirror stack
+    needs: detect
+    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: e2e-changed
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      jaeger:
+        image: jaegertracing/jaeger:2.10.0
+        ports:
+          - 4318:4318
+          - 16686:16686
+    steps:
+      - name: Validate configuration
+        env:
+          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen \
+            --extra proxy --extra proxy-runtime --extra extra_proxy \
+            --extra semantic-router --extra bedrock-realtime \
+            --group ci --group proxy-dev --group e2e-dev
+          uv pip install "pipecat-ai[openai]==1.4.0"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright chromium
+        run: uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure AWS credentials
+        id: aws
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: litellm-e2e-changed-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
+
+      - name: Fetch provider credentials from AWS Secrets Manager
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          umask 077
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
+            --query SecretString --output text \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
+            --query SecretString --output text \
+            | jq -R -s '{"LITELLM_LICENSE": .}' \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+
+      - name: Boot the stage-mirror stack
+        id: boot
+        run: |
+          umask 077
+          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
+            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
+            exit 1
+          fi
+
+      - name: Export stack environment
+        run: |
+          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
+          echo "::add-mask::${master_key}"
+          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
+
+      - name: Run the selected tests three times
+        env:
+          TESTS: ${{ needs.detect.outputs.tests }}
+          E2E_FIXTURE_MODE: live
+        run: |
+          umask 077
+          read -r -a test_files <<< "${TESTS}"
+          for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
+            echo "::group::pass ${pass} of 3"
+            set +e
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
+            status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
+            set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
+            echo "::endgroup::"
+            if [ "${status}" = "5" ]; then
+              echo "::error::the selected files collected no runnable tests, so nothing was verified"
+              exit 1
+            fi
+            if [ "${status}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
+              exit "${status}"
+            fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
+            echo "pass ${pass} of 3 passed"
+          done
+
+      - name: Stop the stack
+        if: always() && steps.boot.outcome != 'skipped'
+        run: bash .github/e2e-stack/down.sh
+
+      - name: Remove credentials and raw output
+        if: always()
+        run: |
+          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+
+  gate:
+    name: e2e-changed-tests
+    needs: [detect, run]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require three successful passes when tests changed
+        env:
+          DETECT_RESULT: ${{ needs.detect.result }}
+          ANY_TESTS: ${{ needs.detect.outputs.any }}
+          RUN_RESULT: ${{ needs.run.result }}
+        run: |
+          if [ "${DETECT_RESULT}" != "success" ]; then
+            echo "::error::changed-test detection did not succeed"
+            exit 1
+          fi
+          if [ "${ANY_TESTS}" = "false" ]; then
+            echo "no changed e2e test files supported by this stack; nothing to run"
+            exit 0
+          fi
+          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
+            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
+            exit 1
+          fi

ci_cd/generate_model_prices_schema.py

@@ -58,6 +58,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
 }
 
 ARRAY_KEYS: dict[str, JsonSchema] = {
+    "supported_audio_formats": {
+        "type": "array",
+        "description": "Audio container formats the model can return.",
+        "items": {"type": "string", "enum": ["mp3", "wav"]},
+    },
     "supported_endpoints": {
         "type": "array",
         "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
@@ -231,6 +236,10 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
         },
         "comment": STRING,
         "audio_transcription_config": STRING,
+        "vertex_ai_audio_api": {
+            "type": "string",
+            "enum": ["lyria_predict", "lyria_interactions"],
+        },
     }
 
 

litellm/cost_calculator.py

@@ -81,6 +81,7 @@ from litellm.llms.together_ai.cost_calculator import (
     get_model_params_and_category,
     has_together_registry_pricing,
 )
+from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost
 from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_character as google_cost_per_character,
 )
@@ -496,6 +497,13 @@ def cost_per_token(
 
     # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models
     if call_type == "speech" or call_type == "aspeech":
+        lyria_generation_cost: Final = (
+            get_vertex_ai_lyria_generation_cost(model=model_without_prefix)
+            if custom_llm_provider in ("vertex_ai", "vertex_ai_beta")
+            else None
+        )
+        if lyria_generation_cost is not None:
+            return 0.0, lyria_generation_cost
         speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider)
         cost_metric: Final = select_cost_metric_for_model(speech_model_info)
         prompt_cost: float = 0.0

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -11,6 +11,7 @@ from typing import (
     Final,
     Literal,
     Protocol,
+    cast,  # noqa: TID251  # rebuilt message_delta dict spans the ContentBlockDelta/MessageBlockDelta union
     get_args,
 )
 
@@ -100,6 +101,10 @@ class _CombinedChunkSplitter:
     @staticmethod
     def _is_combined(chunk: "ModelResponseStream") -> bool:
         """True if ``chunk`` carries response content AND a finish_reason."""
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
+
         choices: Final = _optional_attr_sequence(chunk, "choices")
         if not choices:
             return False
@@ -114,6 +119,7 @@ class _CombinedChunkSplitter:
             or _optional_attr(delta, "tool_calls")
             or _optional_attr(delta, "reasoning_content")
             or _optional_attr(delta, "thinking_blocks")
+            or openai_chat_refusal_text(delta)
         )
 
     _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@@ -305,6 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         # Synthesized compaction block from compact_20260112 polyfill (streaming).
         self.compaction_block = compaction_block
         self.iterations_usage = iterations_usage
+        self._refusal_text: str = ""
         self.sent_compaction_block: bool = False
         # Per-phase flags so the compaction block's start/delta/stop events
         # are emitted (and the public state machine is advanced) in
@@ -572,6 +579,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
+                processed_chunk = self._with_refusal_stop_details(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -806,6 +814,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
+                processed_chunk = self._with_refusal_stop_details(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -993,6 +1002,31 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     def _increment_content_block_index(self):
         self.current_content_block_index += 1
 
+    def _with_refusal_stop_details(
+        self,
+        processed_chunk: ContentBlockDelta | MessageBlockDelta,
+    ) -> ContentBlockDelta | MessageBlockDelta:
+        if processed_chunk.get("type") != "message_delta" or not self._refusal_text:
+            return processed_chunk
+        delta: Final = cast(Mapping[str, object], processed_chunk["delta"])  # cast-ok: keys checked before use
+        if delta.get("stop_reason") == "max_tokens":
+            return processed_chunk
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            refusal_stop_details,
+        )
+
+        return cast(  # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch
+            ContentBlockDelta | MessageBlockDelta,
+            {  # mutable-ok: fresh translation payload; never mutated after construction
+                **processed_chunk,
+                "delta": {  # mutable-ok: fresh message_delta payload; never mutated after construction
+                    **delta,
+                    "stop_reason": "refusal",
+                    "stop_details": refusal_stop_details(self._refusal_text),
+                },
+            },
+        )
+
     @staticmethod
     def _delta_has_content(processed_chunk: Mapping[str, object]) -> bool:
         """Return True if a translated chunk carries a non-empty
@@ -1035,6 +1069,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
         from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
 
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
@@ -1044,6 +1081,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "content", None):
             return False
+        if openai_chat_refusal_text(delta):
+            return False
         if getattr(delta, "reasoning_content", None):
             return False
         # thinking_blocks whose entries are all empty AND unsigned must not
@@ -1067,13 +1106,19 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         - Different content types in the response
         - Specific markers in the content
         """
+        from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+            openai_chat_refusal_text,
+        )
+
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
-        # Example logic - customize based on your needs:
-        # If chunk indicates a tool call
         if chunk.choices[0].finish_reason is not None:
             return False
 
+        refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
+        if refusal_text is not None:
+            self._refusal_text = self._refusal_text + refusal_text
+
         (
             block_type,
             content_block_start,

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -117,6 +117,10 @@ from litellm.llms.anthropic.common_utils import (
 from litellm.llms.anthropic.experimental_pass_through.context_management import (
     PolyfillResult,
 )
+from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+    openai_chat_refusal_text,
+    refusal_stop_details,
+)
 from litellm.types.llms.anthropic import (
     ANTHROPIC_HOSTED_TOOLS,
     AllAnthropicPassThroughMessageValues,
@@ -1314,6 +1318,8 @@ class LiteLLMAnthropicMessagesAdapter:
                 new_content.append(
                     AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
                 )
+            if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
+                new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
             # Handle tool calls (in parallel to text content)
             if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
                 for tool_call in choice.message.tool_calls:
@@ -1472,14 +1478,23 @@ class LiteLLMAnthropicMessagesAdapter:
             choices=response.choices,
             tool_name_mapping=tool_name_mapping,
         )
+        refusal_text: Final = next(
+            (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
+            None,
+        )
 
         if polyfill_result is not None and polyfill_result.compaction_block is not None:
             anthropic_content.insert(0, polyfill_result.compaction_block)
 
         ## extract finish reason
-        anthropic_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
+        translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
             openai_finish_reason=response.choices[0].finish_reason
         )
+        anthropic_finish_reason: Final = (
+            "refusal"
+            if refusal_text is not None and translated_finish_reason != "max_tokens"
+            else translated_finish_reason
+        )
         # extract usage
         usage: Final[Usage] = getattr(response, "usage")
         anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
@@ -1501,6 +1516,7 @@ class LiteLLMAnthropicMessagesAdapter:
             usage=anthropic_usage,
             content=anthropic_content,
             stop_reason=anthropic_finish_reason,
+            stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
         )
 
         applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
@@ -1541,7 +1557,9 @@ class LiteLLMAnthropicMessagesAdapter:
                         "signature": thought_sig,
                     }
                 return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
-            elif choice.delta.content is not None and len(choice.delta.content) > 0:
+            elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
+                choice.delta
+            ) is not None:
                 return "text", TextBlock(type="text", text="")
             elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
                 thinking_blocks = choice.delta.thinking_blocks or []
@@ -1613,7 +1631,10 @@ class LiteLLMAnthropicMessagesAdapter:
         elif reasoning_content:
             return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
         else:
-            return "text_delta", ContentTextBlockDelta(type="text_delta", text=text)
+            refusal_text: Final = "".join(
+                refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
+            )
+            return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)
 
     def translate_streaming_openai_response_to_anthropic(
         self,

litellm/llms/anthropic/experimental_pass_through/messages/utils.py

@@ -1,8 +1,11 @@
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping, Sequence
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints
 
-from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams
+from litellm.types.llms.anthropic import (
+    AnthropicMessagesRequestOptionalParams,
+    AnthropicStopDetails,
+)
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
@@ -25,6 +28,69 @@ def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] |
     return stop_details if isinstance(stop_details, dict) else None
 
 
+def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails:
+    """The ``stop_details`` object accompanying a translated ``stop_reason: "refusal"``."""
+    return AnthropicStopDetails(type="refusal", category=None, explanation=explanation)
+
+
+def _mapping_field(container: object, key: str) -> object | None:
+    """One key of a raw provider payload, or None when the payload is not a mapping."""
+    if not isinstance(container, Mapping):
+        return None
+    return cast(Mapping[str, object], container).get(key)  # cast-ok: raw payload, callers re-check every value
+
+
+def _mapping_str_field(container: object, key: str) -> str | None:
+    value: Final = _mapping_field(container, key)
+    return value if isinstance(value, str) and value else None
+
+
+def openai_chat_refusal_text(message_or_delta: object) -> str | None:
+    """
+    Refusal text carried by an OpenAI Chat Completions message or streaming delta,
+    read from ``refusal`` or from the ``provider_specific_fields`` LiteLLM parks it
+    in, or None when the turn is not a refusal.
+    """
+    refusal: Final = getattr(message_or_delta, "refusal", None)
+    if isinstance(refusal, str) and refusal:
+        return refusal
+    return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
+
+
+def _responses_message_refusal_text(item: object) -> str | None:
+    from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
+
+    if isinstance(item, ResponseOutputMessage):
+        return next(
+            (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal) and part.refusal),
+            None,
+        )
+    raw_parts: Final = _mapping_field(item, "content")
+    if _mapping_str_field(item, "type") != "message" or not isinstance(raw_parts, Sequence):
+        return None
+    return next(
+        (
+            refusal
+            for part in cast(Sequence[object], raw_parts)  # cast-ok: members re-validated below
+            if _mapping_str_field(part, "type") == "refusal"
+            and isinstance(refusal := _mapping_str_field(part, "refusal"), str)
+        ),
+        None,
+    )
+
+
+def responses_output_refusal_text(output: Iterable[object]) -> str | None:
+    """
+    Refusal text carried by an OpenAI Responses ``output`` list, in typed
+    (``ResponseOutputRefusal``) or raw-dictionary shape, or None when none of the
+    output messages refused.
+    """
+    return next(
+        (text for item in output if (text := _responses_message_refusal_text(item)) is not None),
+        None,
+    )
+
+
 def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError":
     """The exception a safeguard-refused Anthropic response converts into so the
     content-policy fallback chain can re-dispatch it."""

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -1,13 +1,18 @@
 # What is this?
 ## Translates OpenAI call to Anthropic `/v1/messages` format
+import asyncio
 import json
 import traceback
 from collections import deque
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+    refusal_stop_details,
+    responses_output_refusal_text,
+)
 from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
 
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
@@ -49,6 +54,8 @@ class AnthropicResponsesStreamWrapper:
         self._sent_message_start = False
         self._sent_message_stop = False
         self._chunk_queue: deque[dict[str, object]] = deque()
+        self._refusal_text: str = ""
+        self._sync_responses_iterator: Iterator[object] | None = None
 
     def _make_message_start(self) -> dict[str, object]:
         return {
@@ -131,6 +138,24 @@ class AnthropicResponsesStreamWrapper:
                 )
             return
 
+        if event_type == "response.refusal.delta":
+            delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
+            if not isinstance(delta, str) or not delta:
+                return
+            self._refusal_text = self._refusal_text + delta
+            item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
+            block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
+            if block_idx < 0:
+                block_idx = self._open_block(item_id, {"type": "text", "text": ""})
+            self._chunk_queue.append(
+                {
+                    "type": "content_block_delta",
+                    "index": block_idx,
+                    "delta": {"type": "text_delta", "text": delta},
+                }
+            )
+            return
+
         # ---- text delta ----
         if event_type == "response.output_text.delta":
             item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
@@ -215,34 +240,47 @@ class AnthropicResponsesStreamWrapper:
             response_obj: Final = getattr(event, "response", None) or (
                 event.get("response") if isinstance(event, dict) else None
             )
-            stop_reason = "end_turn"
-            anthropic_usage: AnthropicUsage = AnthropicUsage(input_tokens=0, output_tokens=0)
-
-            if response_obj is not None:
-                status: Final = getattr(response_obj, "status", None)
-                if status == "incomplete":
-                    stop_reason = "max_tokens"
-                anthropic_usage = (
-                    LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage(
-                        getattr(response_obj, "usage", None)
-                    )
+            output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
+            refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None)
+            status: Final = getattr(response_obj, "status", None) if response_obj is not None else None
+            has_tool_call: Final = any(
+                getattr(item, "type", None) == "function_call"
+                or (isinstance(item, dict) and item.get("type") == "function_call")
+                for item in output
+            )
+            stop_reason: Final = (
+                "max_tokens"
+                if status == "incomplete"
+                else "refusal"
+                if refusal_text is not None
+                else "tool_use"
+                if has_tool_call
+                else "end_turn"
+            )
+            anthropic_usage: Final[AnthropicUsage] = (
+                LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage(
+                    getattr(response_obj, "usage", None)
                 )
+                if response_obj is not None
+                else AnthropicUsage(input_tokens=0, output_tokens=0)
+            )
 
-            # Check if tool_use was in the output to override stop_reason
-            if response_obj is not None:
-                output: Final = getattr(response_obj, "output", []) or []
-                for out_item in output:
-                    out_type = getattr(out_item, "type", None) or (
-                        out_item.get("type") if isinstance(out_item, dict) else None
-                    )
-                    if out_type == "function_call":
-                        stop_reason = "tool_use"
-                        break
+            message_delta_payload: Final = {  # mutable-ok: fresh message_delta payload built per chunk
+                "stop_reason": stop_reason,
+                "stop_sequence": None,
+                **(
+                    {  # mutable-ok: fresh message_delta stop_details entry built per chunk
+                        "stop_details": refusal_stop_details(refusal_text)
+                    }
+                    if stop_reason == "refusal"
+                    else {}  # mutable-ok: empty spread placeholder for non-refusal stop
+                ),
+            }
 
             self._chunk_queue.append(
                 {
                     "type": "message_delta",
-                    "delta": {"stop_reason": stop_reason, "stop_sequence": None},
+                    "delta": message_delta_payload,
                     "usage": dict(anthropic_usage),
                 }
             )
@@ -266,10 +304,20 @@ class AnthropicResponsesStreamWrapper:
 
         # Consume the upstream stream
         try:
-            async for event in self.responses_stream:
-                self._process_event(event)
-                if self._chunk_queue:
-                    return self._chunk_queue.popleft()
+            if hasattr(self.responses_stream, "__aiter__"):
+                async for event in self.responses_stream:
+                    self._process_event(event)
+                    if self._chunk_queue:
+                        return self._chunk_queue.popleft()
+            else:
+                if self._sync_responses_iterator is None:
+                    self._sync_responses_iterator = iter(self.responses_stream)
+                sync_iterator: Final = self._sync_responses_iterator
+                missing: Final = object()
+                while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing:
+                    self._process_event(event)
+                    if self._chunk_queue:
+                        return self._chunk_queue.popleft()
         except StopAsyncIteration:
             pass
         except Exception as e:

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -19,6 +19,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
+from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
+    refusal_stop_details,
+    responses_output_refusal_text,
+)
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     prompt_cache_key_from_user_id,
@@ -624,6 +628,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
 
         content: Final[list[dict[str, object]]] = []
         stop_reason: AnthropicFinishReason = "end_turn"
+        refusal_text: Final = responses_output_refusal_text(
+            cast(Iterable[object], response.output)  # cast-ok: output items re-validated per item
+        )
 
         for item in response.output:
             if isinstance(item, ResponseReasoningItem):
@@ -631,10 +638,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
 
             elif isinstance(item, ResponseOutputMessage):
                 for part in item.content:
-                    if getattr(part, "type", None) == "output_text":
+                    part_type = getattr(part, "type", None)
+                    if part_type == "output_text":
                         content.append(
                             AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump()
                         )
+                    elif part_type == "refusal":
+                        content.append(
+                            AnthropicResponseContentBlockText(
+                                type="text", text=getattr(part, "refusal", "") or ""
+                            ).model_dump()
+                        )
 
             elif isinstance(item, ResponseFunctionToolCall):
                 try:
@@ -654,11 +668,21 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             elif isinstance(item, dict):
                 item_type = item.get("type")
                 if item_type == "message":
-                    for part in item.get("content", []):
-                        if isinstance(part, dict) and part.get("type") == "output_text":
-                            content.append(
-                                AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump()
-                            )
+                    for part in item.get("content", ()):
+                        if isinstance(part, dict):
+                            part_type = part.get("type")
+                            if part_type == "output_text":
+                                content.append(
+                                    AnthropicResponseContentBlockText(
+                                        type="text", text=part.get("text", "")
+                                    ).model_dump()
+                                )
+                            elif part_type == "refusal":
+                                content.append(
+                                    AnthropicResponseContentBlockText(
+                                        type="text", text=part.get("refusal", "") or ""
+                                    ).model_dump()
+                                )
                 elif item_type == "reasoning":
                     content.extend(
                         self._thinking_blocks_from_reasoning_item(
@@ -679,10 +703,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         ).model_dump(exclude_none=True)
                     )
                     stop_reason = "tool_use"
-
-        # status -> stop_reason override
         if response.status == "incomplete":
             stop_reason = "max_tokens"
+        elif refusal_text is not None:
+            stop_reason = "refusal"
 
         anthropic_usage: Final = self.translate_responses_api_usage_to_anthropic_usage(response.usage)
 
@@ -695,4 +719,5 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             usage=anthropic_usage,
             content=content,
             stop_reason=stop_reason,
+            stop_details=(refusal_stop_details(refusal_text) if stop_reason == "refusal" else None),
         )

litellm/llms/vertex_ai/common_utils.py

@@ -1,9 +1,14 @@
 import re
+from collections.abc import Mapping
 from copy import deepcopy
 from enum import Enum
+from functools import lru_cache
+from types import MappingProxyType
 from typing import Any, Final, Literal, cast, get_type_hints
 
 import httpx
+from pydantic import TypeAdapter, ValidationError
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -21,6 +26,61 @@ from litellm.types.utils import TokenCountResponse
 from litellm.utils import supports_response_schema, supports_system_messages
 
 
+class VertexAILyriaModelInfo(TypedDict):
+    vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"]]
+    supported_audio_formats: ReadOnly[tuple[Literal["mp3", "wav"], ...]]
+    output_cost_per_image: NotRequired[ReadOnly[float]]
+
+
+_VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER: Final = TypeAdapter(VertexAILyriaModelInfo)
+
+
+def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyriaModelInfo | None:
+    if raw_model_info is None:
+        return None
+    try:
+        return _VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER.validate_python(raw_model_info)
+    except ValidationError:
+        return None
+
+
+@lru_cache(maxsize=1)
+def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]:
+    from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
+
+    return MappingProxyType(
+        {
+            model_key: lyria_model_info
+            for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items()
+            if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None
+        }
+    )
+
+
+def _vertex_ai_lyria_model_key(model: str) -> str:
+    return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
+
+
+def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None:
+    return None if model_info is None else model_info.get("output_cost_per_image")
+
+
+def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None:
+    model_key: Final = _vertex_ai_lyria_model_key(model)
+    runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
+    return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key)
+
+
+def get_vertex_ai_lyria_generation_cost(model: str) -> float | None:
+    model_key: Final = _vertex_ai_lyria_model_key(model)
+    runtime_cost: Final = _vertex_ai_lyria_generation_cost(
+        _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
+    )
+    if runtime_cost is not None:
+        return runtime_cost
+    return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key))
+
+
 class VertexAIError(BaseLLMException):
     def __init__(
         self,

litellm/llms/vertex_ai/text_to_speech/transformation.py

@@ -8,17 +8,25 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s
 import base64
 from collections.abc import Coroutine
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union
 
 import httpx
 
+import litellm
+from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.audio_utils.utils import (
+    DEFAULT_SPEECH_MEDIA_TYPE,
     speech_media_type_from_audio_bytes,
 )
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.text_to_speech.transformation import (
     BaseTextToSpeechConfig,
     TextToSpeechRequestData,
 )
+from litellm.llms.vertex_ai.common_utils import (
+    VertexAILyriaModelInfo,
+    get_vertex_ai_lyria_model_info,
+)
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
 from litellm.types.llms.vertex_ai_text_to_speech import (
@@ -35,6 +43,10 @@ else:
     LiteLLMLoggingObj = Any
     HttpxBinaryResponseContent = Any
 
+_LyriaVoice: TypeAlias = (
+    str | dict | None
+)  # mutable-ok: inherited interface supports structured provider voice dictionaries
+
 
 class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
     """
@@ -472,3 +484,209 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
 
         # Initialize the HttpxBinaryResponseContent instance
         return HttpxBinaryResponseContent(response)
+
+
+class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig):
+    @classmethod
+    def is_lyria_model(cls, model: str) -> bool:
+        return get_vertex_ai_lyria_model_info(model=model) is not None
+
+    @staticmethod
+    def _get_model_info(model: str) -> VertexAILyriaModelInfo:
+        model_info: Final = get_vertex_ai_lyria_model_info(model=model)
+        if model_info is None:
+            raise ValueError(f"Vertex AI model {model!r} does not declare a Lyria audio API")
+        return model_info
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list:  # mutable-ok: inherited provider interface returns a concrete parameter list
+        return [  # mutable-ok: inherited provider interface requires a concrete parameter list
+            "response_format"
+        ]
+
+    def map_openai_params(
+        self,
+        model: str,
+        optional_params: dict,  # mutable-ok: inherited provider interface accepts a concrete parameter dictionary
+        voice: _LyriaVoice = None,
+        drop_params: bool = False,
+        kwargs: dict | None = None,  # mutable-ok: inherited provider interface accepts a concrete keyword dictionary
+    ) -> tuple[str | None, dict]:  # mutable-ok: inherited provider interface returns concrete mapped parameters
+        mapped_params: Final = dict(  # mutable-ok: mapping drops unsupported parameters before provider dispatch
+            optional_params
+        )
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        unsupported_params: Final = tuple(
+            param for param in ("speed", "instructions") if mapped_params.get(param) is not None
+        )
+        if unsupported_params:
+            if drop_params or litellm.drop_params:
+                for param in unsupported_params:
+                    mapped_params.pop(param, None)
+            else:
+                raise UnsupportedParamsError(
+                    status_code=400,
+                    message=(
+                        f"Vertex AI {base_model} does not support the OpenAI parameters: "
+                        f"{', '.join(unsupported_params)}. To drop unsupported openai params "
+                        "from the call, set `litellm.drop_params = True`"
+                    ),
+                )
+        response_format: Final = mapped_params.get("response_format")
+        supported_formats: Final = frozenset(model_info["supported_audio_formats"])
+        if response_format is not None and response_format not in supported_formats:
+            if drop_params or litellm.drop_params:
+                mapped_params.pop("response_format", None)
+            else:
+                raise UnsupportedParamsError(
+                    status_code=400,
+                    message=(
+                        f"Vertex AI {base_model} does not support response_format={response_format!r}. "
+                        f"Supported values: {', '.join(sorted(supported_formats))}. "
+                        "To drop unsupported openai params from the call, set `litellm.drop_params = True`"
+                    ),
+                )
+        return voice if isinstance(voice, str) else None, mapped_params
+
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: str | None,
+        litellm_params: dict,  # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters
+    ) -> str:
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        configured_project: Final = self.safe_get_vertex_ai_project(litellm_params)
+        project: Final = (
+            self._ensure_access_token(
+                credentials=self.safe_get_vertex_ai_credentials(litellm_params),
+                project_id=None,
+                custom_llm_provider="vertex_ai",
+            )[1]
+            if configured_project is None
+            else configured_project
+        )
+        if model_info["vertex_ai_audio_api"] == "lyria_interactions":
+            from litellm.llms.vertex_ai.interactions.transformation import (
+                VertexAIInteractionsConfig,
+            )
+
+            def mint_access_token(
+                _credentials: VERTEX_CREDENTIALS_TYPES | None,
+                project_id: str | None,
+            ) -> tuple[str, str]:
+                return "", project_id or project
+
+            return VertexAIInteractionsConfig(mint_access_token=mint_access_token).get_complete_url(
+                api_base=api_base,
+                model=base_model,
+                litellm_params={  # mutable-ok: interactions dispatch expects a concrete parameter dictionary
+                    **litellm_params,
+                    "vertex_project": project,
+                    "vertex_location": "global",
+                },
+            )
+        location: Final = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location()
+        base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/")
+        encoded_project: Final = encode_url_path_segment(project, field_name="project")
+        encoded_location: Final = encode_url_path_segment(location, field_name="location")
+        encoded_model: Final = encode_url_path_segment(base_model, field_name="model")
+        return (
+            f"{base_url}/v1/projects/{encoded_project}/locations/{encoded_location}"
+            f"/publishers/google/models/{encoded_model}:predict"
+        )
+
+    def transform_text_to_speech_request(
+        self,
+        model: str,
+        input: str,
+        voice: str | None,
+        optional_params: dict,  # mutable-ok: inherited provider interface accepts concrete mapped parameters
+        litellm_params: dict,  # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters
+        headers: dict,  # mutable-ok: inherited provider interface accepts and updates concrete HTTP headers
+    ) -> TextToSpeechRequestData:
+        access_token, project = self._ensure_access_token(
+            credentials=self.safe_get_vertex_ai_credentials(litellm_params),
+            project_id=self.safe_get_vertex_ai_project(litellm_params),
+            custom_llm_provider="vertex_ai",
+        )
+        headers.update(
+            {  # mutable-ok: HTTP dispatch requires a concrete header dictionary
+                "Authorization": f"Bearer {access_token}",
+                "x-goog-user-project": project,
+                "Content-Type": "application/json",
+            }
+        )
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        request_body: Final[dict[str, object]] = (  # mutable-ok: HTTP dispatch requires a concrete provider payload
+            {  # mutable-ok: predict dispatch requires a concrete provider request dictionary
+                "instances": [  # mutable-ok: predict dispatch requires a concrete instances list
+                    {"prompt": input}  # mutable-ok: predict dispatch requires a concrete instance dictionary
+                ],
+                "parameters": {  # mutable-ok: predict dispatch requires a concrete parameters dictionary
+                    "sample_count": 1
+                },
+            }
+            if model_info["vertex_ai_audio_api"] == "lyria_predict"
+            else {  # mutable-ok: interactions dispatch requires a concrete provider request dictionary
+                "model": base_model,
+                "input": input,
+                **(
+                    {  # mutable-ok: interactions dispatch requires a nested response-format dictionary
+                        "response_format": {  # mutable-ok: interactions response format is a concrete provider payload
+                            "type": "audio",
+                            "mime_type": "audio/wav",
+                        }
+                    }
+                    if optional_params.get("response_format") == "wav"
+                    else {}  # mutable-ok: no response override is merged for non-WAV output
+                ),
+            }
+        )
+        return TextToSpeechRequestData(dict_body=request_body, headers=headers)
+
+    def transform_text_to_speech_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "LiteLLMLoggingObj",
+    ) -> "HttpxBinaryResponseContent":
+        from litellm.types.llms.openai import HttpxBinaryResponseContent
+
+        response_json: Final = raw_response.json()
+        base_model: Final = model.removeprefix("vertex_ai/")
+        model_info: Final = self._get_model_info(model=model)
+        audio_data: str | None = None  # rebind-ok: response parsing discovers audio data in provider-specific shapes
+        mime_type: str | None = None  # rebind-ok: response parsing discovers the MIME type beside the audio payload
+        if model_info["vertex_ai_audio_api"] == "lyria_predict":
+            predictions: Final = response_json.get("predictions") or ()
+            if predictions:
+                audio_data = predictions[0].get("audioContent") or predictions[0].get(
+                    "bytesBase64Encoded"
+                )  # rebind-ok: predict response supplies the generated audio value
+                mime_type = predictions[0].get("mimeType")  # rebind-ok: predict response supplies its audio MIME type
+        else:
+            for step in response_json.get("steps") or response_json.get("outputs") or ():
+                content_items = step.get("content") or () if step.get("type") == "model_output" else (step,)
+                for content in content_items:
+                    if content.get("type") == "audio" and content.get("data"):
+                        audio_data = content[
+                            "data"
+                        ]  # rebind-ok: interactions response supplies the generated audio value
+                        mime_type = content.get(
+                            "mime_type"
+                        )  # rebind-ok: interactions response supplies its audio MIME type
+        if audio_data is None:
+            raise ValueError(f"No generated audio found in Vertex AI {base_model} response")
+        binary_data: Final = base64.b64decode(audio_data)
+        media_type: Final = mime_type or speech_media_type_from_audio_bytes(binary_data) or DEFAULT_SPEECH_MEDIA_TYPE
+        return HttpxBinaryResponseContent(
+            httpx.Response(
+                status_code=raw_response.status_code,
+                content=binary_data,
+                headers=MappingProxyType({"content-type": media_type}),
+            )
+        )

litellm/main.py

@@ -8247,6 +8247,7 @@ def speech(
         )
     elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
         from litellm.llms.vertex_ai.text_to_speech.transformation import (
+            VertexAILyriaTextToSpeechConfig,
             VertexAITextToSpeechConfig,
         )
 
@@ -8271,7 +8272,11 @@ def speech(
 
         # Vertex AI Text-to-Speech (Google Cloud TTS)
         if text_to_speech_provider_config is None:
-            text_to_speech_provider_config = VertexAITextToSpeechConfig()
+            text_to_speech_provider_config = (  # rebind-ok: model metadata selects the Vertex TTS implementation
+                VertexAILyriaTextToSpeechConfig()
+                if VertexAILyriaTextToSpeechConfig.is_lyria_model(model)
+                else VertexAITextToSpeechConfig()
+            )
 
         # Cast to specific Vertex AI config type to access dispatch method
         vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config)

litellm/model_prices_and_context_window_backup.json

@@ -47141,6 +47141,99 @@
         "output_cost_per_token": 4e-07,
         "supports_tool_choice": true
     },
+    "vertex_ai/lyria-002": {
+        "litellm_provider": "vertex_ai",
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.06,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "wav"
+        ],
+        "supported_endpoints": [
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supports_audio_output": true,
+        "vertex_ai_audio_api": "lyria_predict"
+    },
+    "vertex_ai/lyria-3-clip-preview": {
+        "input_cost_per_token": 0,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.04,
+        "output_cost_per_token": 0,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "mp3"
+        ],
+        "supported_endpoints": [
+            "/v1beta/interactions",
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supported_regions": [
+            "global"
+        ],
+        "supports_audio_input": false,
+        "supports_audio_output": true,
+        "supports_function_calling": false,
+        "supports_prompt_caching": false,
+        "supports_response_schema": false,
+        "supports_system_messages": false,
+        "supports_vision": false,
+        "supports_web_search": false,
+        "vertex_ai_audio_api": "lyria_interactions"
+    },
+    "vertex_ai/lyria-3-pro-preview": {
+        "input_cost_per_token": 0,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.08,
+        "output_cost_per_token": 0,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "mp3",
+            "wav"
+        ],
+        "supported_endpoints": [
+            "/v1beta/interactions",
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supported_regions": [
+            "global"
+        ],
+        "supports_audio_input": false,
+        "supports_audio_output": true,
+        "supports_function_calling": false,
+        "supports_prompt_caching": false,
+        "supports_response_schema": false,
+        "supports_system_messages": false,
+        "supports_vision": false,
+        "supports_web_search": false,
+        "vertex_ai_audio_api": "lyria_interactions"
+    },
     "vertex_ai/meta/llama-3.1-405b-instruct-maas": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "vertex_ai-llama_models",

litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py

@@ -10,7 +10,10 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.vertex_ai.common_utils import get_vertex_location_from_url
+from litellm.llms.vertex_ai.common_utils import (
+    get_vertex_ai_lyria_generation_cost,
+    get_vertex_location_from_url,
+)
 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
     ModelResponseIterator as VertexModelResponseIterator,
 )
@@ -44,7 +47,6 @@ else:
     PassThroughEndpointLogging = Any
     LiteLLMBatch = Any
 
-# Define EndpointType locally to avoid import issues
 EndpointType = Any
 
 
@@ -270,6 +272,16 @@ class VertexPassthroughLoggingHandler:
         _json_response: Final[dict[str, object]] = httpx_response.json()
 
         litellm_prediction_response: ModelResponse | EmbeddingResponse | ImageResponse = ModelResponse()
+        if VertexPassthroughLoggingHandler._is_audio_predict_response(
+            model=model,
+            json_response=_json_response,
+        ):
+            return VertexPassthroughLoggingHandler._handle_audio_predict_response(
+                json_response=_json_response,
+                logging_obj=logging_obj,
+                model=model,
+                kwargs=kwargs,
+            )
         if vertex_image_generation_class.is_image_generation_response(_json_response):
             litellm_prediction_response = vertex_image_generation_class.process_image_generation_response(
                 _json_response,
@@ -323,6 +335,71 @@ class VertexPassthroughLoggingHandler:
             "kwargs": kwargs,
         }
 
+    @staticmethod
+    def _handle_audio_predict_response(
+        json_response: dict,  # mutable-ok: passthrough logging receives the decoded provider response dictionary
+        logging_obj: LiteLLMLoggingObj,
+        model: str,
+        kwargs: dict,  # mutable-ok: passthrough logging enriches the shared callback metadata dictionary
+    ) -> PassThroughEndpointLoggingTypedDict:
+        prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count(
+            json_response=json_response
+        )
+        response_cost: Final = (get_vertex_ai_lyria_generation_cost(model=model) or 0.0) * prediction_count
+
+        logging_obj.model = model  # rebind-ok: passthrough attribution records the resolved Vertex model
+        logging_obj.model_call_details[  # rebind-ok: passthrough attribution enriches callback metadata
+            "model"
+        ] = model
+        logging_obj.model_call_details[  # rebind-ok: passthrough attribution enriches callback metadata
+            "custom_llm_provider"
+        ] = "vertex_ai"
+        logging_obj.custom_llm_provider = (  # rebind-ok: attribution records the resolved provider
+            "vertex_ai"
+        )
+        logging_obj.model_call_details[  # rebind-ok: passthrough attribution enriches callback metadata
+            "response_cost"
+        ] = response_cost
+
+        kwargs[  # rebind-ok: callback metadata is enriched for downstream hooks
+            "response_cost"
+        ] = response_cost
+        kwargs["model"] = model  # rebind-ok: callback metadata records the resolved model
+        kwargs["custom_llm_provider"] = "vertex_ai"  # rebind-ok: callback metadata records the resolved provider
+
+        standard_pass_through_response_object: Final[
+            StandardPassThroughResponseObject
+        ] = {  # mutable-ok: callback contract requires a concrete response dictionary
+            "response": json_response,
+        }
+        return {  # mutable-ok: passthrough logging contract requires a concrete result dictionary
+            "result": standard_pass_through_response_object,
+            "kwargs": kwargs,
+        }
+
+    @staticmethod
+    def _is_audio_predict_response(
+        model: str,
+        json_response: dict,  # mutable-ok: predicate inspects the decoded provider response dictionary without mutation
+    ) -> bool:
+        return (
+            VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0
+            and get_vertex_ai_lyria_generation_cost(model=model) is not None
+        )
+
+    @staticmethod
+    def _get_audio_prediction_count(
+        json_response: dict,  # mutable-ok: counter inspects the decoded provider response dictionary without mutation
+    ) -> int:
+        predictions: Final = json_response.get("predictions")
+        if not isinstance(predictions, list):
+            return 0
+        return sum(
+            1
+            for prediction in predictions
+            if isinstance(prediction, dict) and (prediction.get("audioContent") or prediction.get("bytesBase64Encoded"))
+        )
+
     @staticmethod
     def _extract_embed_content_input(request_body: dict | None, batch: bool) -> str:
         """Extract raw input text from an :embedContent or :batchEmbedContents request body for token counting."""

litellm/router_strategy/lowest_cost.py

@@ -1,6 +1,6 @@
 #### What this does ####
 #   picks based on response time (for streaming, this is time to first token)
-from datetime import datetime, timedelta
+from datetime import datetime
 from typing import Final
 
 import litellm
@@ -52,16 +52,12 @@ class LowestCostLoggingHandler(CustomLogger):
                 precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
                 cost_key: Final = f"{model_group}_map"
 
-                response_ms: Final[timedelta] = end_time - start_time
-
                 total_tokens = 0
 
                 if isinstance(response_obj, ModelResponse):
                     _usage: Final = getattr(response_obj, "usage", None)
                     if _usage is not None and isinstance(_usage, litellm.Usage):
-                        completion_tokens: Final = _usage.completion_tokens
                         total_tokens = _usage.total_tokens
-                        float(response_ms.total_seconds() / completion_tokens)
 
                 # ------------
                 # Update usage
@@ -131,18 +127,13 @@ class LowestCostLoggingHandler(CustomLogger):
                 current_minute: Final = datetime.now().strftime("%M")
                 precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
 
-                response_ms: Final[timedelta] = end_time - start_time
-
                 total_tokens = 0
 
                 if isinstance(response_obj, ModelResponse):
                     _usage: Final = getattr(response_obj, "usage", None)
                     if _usage is not None and isinstance(_usage, litellm.Usage):
-                        completion_tokens: Final = _usage.completion_tokens
                         total_tokens = _usage.total_tokens
 
-                        float(response_ms.total_seconds() / completion_tokens)
-
                 # ------------
                 # Update usage
                 # ------------

litellm/router_strategy/lowest_latency.py

@@ -1,6 +1,7 @@
 #### What this does ####
 #   picks based on response time (for streaming, this is time to first token)
 import random
+from collections.abc import Sequence
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Final
 
@@ -25,6 +26,12 @@ class RoutingArgs(LiteLLMPydanticObjectBase):
     max_latency_list_size: int = 10
 
 
+def _average_latency(samples: Sequence[float]) -> float:
+    if not samples:
+        return 0.0
+    return sum(samples) / len(samples)
+
+
 class LowestLatencyLoggingHandler(CustomLogger):
     test_flag: bool = False
     logged_success: int = 0
@@ -431,23 +438,13 @@ class LowestLatencyLoggingHandler(CustomLogger):
             item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
 
             # get average latency or average ttft (depending on streaming/non-streaming)
-            total: float = 0.0
             use_ttft = (
                 request_kwargs is not None
                 and request_kwargs.get("stream", None) is not None
                 and request_kwargs["stream"] is True
                 and len(item_ttft_latency) > 0
             )
-            if use_ttft:
-                for _call_latency in item_ttft_latency:
-                    if isinstance(_call_latency, float):
-                        total += _call_latency
-                item_latency = total / len(item_ttft_latency)
-            else:
-                for _call_latency in item_latency:
-                    if isinstance(_call_latency, float):
-                        total += _call_latency
-                item_latency = total / len(item_latency)
+            average_latency = _average_latency(item_ttft_latency if use_ttft else item_latency)
 
             # -------------- #
             # Debugging Logic
@@ -456,7 +453,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
             # this helps a user to debug why the router picked a specfic deployment      #
             _deployment_api_base = _deployment.get("litellm_params", {}).get("api_base", "")
             if _deployment_api_base is not None:
-                _latency_per_deployment[_deployment_api_base] = item_latency
+                _latency_per_deployment[_deployment_api_base] = average_latency
             # -------------- #
             # End of Debugging Logic
             # -------------- #
@@ -466,7 +463,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
             ):  # if user passed in tpm / rpm in the model_list
                 continue
             else:
-                potential_deployments.append((_deployment, item_latency))
+                potential_deployments.append((_deployment, average_latency))
 
         if len(potential_deployments) == 0:
             return None

litellm/types/llms/anthropic.py

@@ -520,8 +520,15 @@ ContentBlockContentBlockDict = ToolUseBlock | TextBlock | ChatCompletionThinking
 ContentBlockStart = ContentBlockStartToolUse | ContentBlockStartText
 
 
+class AnthropicStopDetails(TypedDict, total=False):
+    type: ReadOnly[Literal["refusal"]]
+    category: ReadOnly[str | None]
+    explanation: ReadOnly[str | None]
+
+
 class MessageDelta(TypedDict, total=False):
     stop_reason: str | None
+    stop_details: ReadOnly[AnthropicStopDetails]
 
 
 class ServerToolUsage(TypedDict, total=False):
@@ -658,7 +665,7 @@ class AnthropicOutputTokensDetails(BaseModel):
     thinking_tokens: int | None = None
 
 
-AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"]
+AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"]
 
 
 class AnthropicResponse(BaseModel):

litellm/types/llms/anthropic_messages/anthropic_response.py

@@ -5,6 +5,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 from litellm.types.llms.anthropic import (
     AnthropicResponseContentBlockText,
     AnthropicResponseContentBlockToolUse,
+    AnthropicStopDetails,
     ContextManagementResponse,
     ServerToolUsage,
 )
@@ -78,16 +79,6 @@ class AnthropicUsage(TypedDict, total=False):
     server_tool_use: NotRequired[ReadOnly[ServerToolUsage]]
 
 
-class AnthropicStopDetails(TypedDict, total=False):
-    """
-    Safeguard verdict accompanying a `stop_reason: "refusal"` response:
-    https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
-    """
-
-    category: ReadOnly[str | None]
-    explanation: ReadOnly[str | None]
-
-
 class AnthropicMessagesResponse(TypedDict, total=False):
     """
     Anthropic Messages API Response: https://docs.anthropic.com/en/api/messages

litellm/types/utils.py

@@ -168,6 +168,8 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
     default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None]
     supports_output_config: bool | None
     supports_image_size: bool | None
+    supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None]
+    vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None]
     bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None
     bedrock_converse_supports_strict_tools: bool | None
 
@@ -335,6 +337,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
             "image_generation",
             "chat",
             "audio_transcription",
+            "audio_speech",
             "responses",
             "ocr",
             "realtime",

litellm/utils.py

@@ -5946,6 +5946,8 @@ def _get_model_info_helper(
                 provider_specific_entry=_model_info.get("provider_specific_entry", None),
                 uses_embed_content=_model_info.get("uses_embed_content", None),
                 supports_image_size=_model_info.get("supports_image_size", None),
+                supported_audio_formats=_model_info.get("supported_audio_formats", None),
+                vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None),
             )
             for cost_key, cost_value in _model_info.items():
                 if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None:
@@ -9436,9 +9438,12 @@ class ProviderConfigManager:
                 # mapping would drop response_format before the bridge sees it (LIT-6501)
                 return None
             from litellm.llms.vertex_ai.text_to_speech.transformation import (
+                VertexAILyriaTextToSpeechConfig,
                 VertexAITextToSpeechConfig,
             )
 
+            if VertexAILyriaTextToSpeechConfig.is_lyria_model(model):
+                return VertexAILyriaTextToSpeechConfig()
             return VertexAITextToSpeechConfig()
         elif litellm.LlmProviders.MINIMAX == provider:
             from litellm.llms.minimax.text_to_speech.transformation import (

model_prices_and_context_window.json

@@ -47141,6 +47141,99 @@
         "output_cost_per_token": 4e-07,
         "supports_tool_choice": true
     },
+    "vertex_ai/lyria-002": {
+        "litellm_provider": "vertex_ai",
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.06,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "wav"
+        ],
+        "supported_endpoints": [
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supports_audio_output": true,
+        "vertex_ai_audio_api": "lyria_predict"
+    },
+    "vertex_ai/lyria-3-clip-preview": {
+        "input_cost_per_token": 0,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.04,
+        "output_cost_per_token": 0,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "mp3"
+        ],
+        "supported_endpoints": [
+            "/v1beta/interactions",
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supported_regions": [
+            "global"
+        ],
+        "supports_audio_input": false,
+        "supports_audio_output": true,
+        "supports_function_calling": false,
+        "supports_prompt_caching": false,
+        "supports_response_schema": false,
+        "supports_system_messages": false,
+        "supports_vision": false,
+        "supports_web_search": false,
+        "vertex_ai_audio_api": "lyria_interactions"
+    },
+    "vertex_ai/lyria-3-pro-preview": {
+        "input_cost_per_token": 0,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "audio_speech",
+        "output_cost_per_image": 0.08,
+        "output_cost_per_token": 0,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
+        "supported_audio_formats": [
+            "mp3",
+            "wav"
+        ],
+        "supported_endpoints": [
+            "/v1beta/interactions",
+            "/v1/audio/speech"
+        ],
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "audio"
+        ],
+        "supported_regions": [
+            "global"
+        ],
+        "supports_audio_input": false,
+        "supports_audio_output": true,
+        "supports_function_calling": false,
+        "supports_prompt_caching": false,
+        "supports_response_schema": false,
+        "supports_system_messages": false,
+        "supports_vision": false,
+        "supports_web_search": false,
+        "vertex_ai_audio_api": "lyria_interactions"
+    },
     "vertex_ai/meta/llama-3.1-405b-instruct-maas": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "vertex_ai-llama_models",

model_prices_and_context_window.schema.json

@@ -623,6 +623,17 @@
           "type": "string",
           "description": "URL of the provider pricing/model page this entry was taken from."
         },
+        "supported_audio_formats": {
+          "type": "array",
+          "description": "Audio container formats the model can return.",
+          "items": {
+            "type": "string",
+            "enum": [
+              "mp3",
+              "wav"
+            ]
+          }
+        },
         "supported_endpoints": {
           "type": "array",
           "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
@@ -846,6 +857,13 @@
         "uses_embed_content": {
           "type": "boolean"
         },
+        "vertex_ai_audio_api": {
+          "type": "string",
+          "enum": [
+            "lyria_predict",
+            "lyria_interactions"
+          ]
+        },
         "web_search_billing_unit": {
           "type": "string",
           "description": "Whether web search is billed per query or per prompt.",

pyproject.toml

@@ -388,7 +388,7 @@ mutate_only_covered_lines = true
 # registered hash algorithm". Nothing to do with mutation coverage, and one
 # erroring test is enough to end the stats phase before any mutant runs.
 pytest_add_cli_args = [
-    "-p", "no:retry",
+    "-p", "no:pytest-retry",
     "-p", "no:rerunfailures",
     "-p", "no:xdist",
     "--ignore=tests/test_litellm/proxy/management_endpoints/test_saml_sso.py",

tests/code_coverage_tests/test_e2e_changed_gate.py

@@ -0,0 +1,203 @@
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
+SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py")
+SELECT_TESTS: Final = GATE.with_name("select_tests.py")
+CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
+SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
+
+
+@pytest.mark.parametrize(
+    ("second_outcome", "expected_status"),
+    (("passed", 0), ("skipped", 1), ("failure", 1), ("error", 1), ("deselected", 1)),
+)
+def test_each_changed_file_must_run(tmp_path: Path, second_outcome: str, expected_status: int) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    if second_outcome != "deselected":
+        second: Final = ET.SubElement(suite, "testcase", file=SELECTED[1])
+        if second_outcome != "passed":
+            _ = ET.SubElement(second, second_outcome)
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == expected_status, result.stdout
+
+
+@pytest.mark.parametrize("outcome", ("failure", "error"))
+def test_passing_case_does_not_hide_a_failure_in_the_same_file(tmp_path: Path, outcome: str) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    failed: Final = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    _ = ET.SubElement(failed, outcome)
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run(
+        [sys.executable, str(GATE), str(report), SELECTED[0]], capture_output=True, text=True
+    )
+
+    assert result.returncode == 1
+
+
+def test_failed_cases_are_named_per_selected_file(tmp_path: Path) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_ok")
+    failed: Final = ET.SubElement(
+        suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_boom"
+    )
+    _ = ET.SubElement(failed, "failure", message="secret-bearing message")
+    errored: Final = ET.SubElement(
+        suite, "testcase", file=SELECTED[1], classname="tests.e2e.access_control.test_b", name="test_setup"
+    )
+    _ = ET.SubElement(errored, "error")
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == 1
+    assert "  failed: tests.e2e.access_control.test_a::test_boom\n" in result.stdout
+    assert "  failed: tests.e2e.access_control.test_b::test_setup\n" in result.stdout
+    assert "test_ok" not in result.stdout
+    assert "secret-bearing message" not in result.stdout
+
+
+@pytest.mark.parametrize("contents", ("<testsuite/>", "<testsuite", '<testsuite><testcase name="a"/></testsuite>'))
+def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None:
+    report: Final = tmp_path / "report.xml"
+    _ = report.write_text(contents)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == 1
+
+
+def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_path: Path) -> None:
+    env_path: Final = tmp_path / ".env"
+
+    result: Final = subprocess.run(
+        [sys.executable, str(SECRETS_TO_ENV), str(env_path)],
+        input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}',
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert result.stdout == "::add-mask::sk-0123456789abcdef\n"
+    assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n"
+
+
+def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
+    result: Final = subprocess.run(
+        [sys.executable, str(SELECT_TESTS), *CANARY],
+        input="".join(f"{path}\n" for path in changed),
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stderr
+    return tuple(result.stdout.split())
+
+
+@pytest.mark.parametrize(
+    ("changed", "expected"),
+    (
+        (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)),
+        (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()),
+        (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()),
+        (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()),
+        (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()),
+        (
+            ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",),
+            ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",),
+        ),
+        (
+            ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",),
+            ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",),
+        ),
+        (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/CLAUDE.md"), ()),
+        (
+            ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"),
+            ("tests/e2e/logging/test_datadog_e2e.py",),
+        ),
+    ),
+)
+def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them(
+    changed: tuple[str, ...], expected: tuple[str, ...]
+) -> None:
+    assert select_tests(changed) == expected
+
+
+@pytest.mark.parametrize(
+    "harness_file",
+    (
+        "tests/e2e/proxy_client.py",
+        "tests/e2e/conftest.py",
+        "tests/e2e/pytest.ini",
+        "tests/e2e/gateway/stage_mirror_ci_config.yml",
+        ".github/e2e-stack/up.sh",
+        ".github/workflows/test-e2e-changed.yml",
+    ),
+)
+def test_harness_changes_run_the_canary_suite(harness_file: str) -> None:
+    assert select_tests((harness_file, "litellm/router.py")) == CANARY
+
+
+def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() -> None:
+    assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY
+
+
+def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None:
+    assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == (
+        *CANARY,
+        "tests/e2e/logging/test_datadog_e2e.py",
+    )
+
+
+def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None:
+    assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py")
+
+
+def test_a_canary_argument_the_shell_never_expanded_fails_the_selector() -> None:
+    result: Final = subprocess.run(
+        [sys.executable, str(SELECT_TESTS), "tests/e2e/access_control/test_*.py"],
+        input="tests/e2e/proxy_client.py\n",
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 1
+    assert "tests/e2e/access_control/test_*.py" in result.stderr
+    assert result.stdout == ""
+
+
+@pytest.mark.parametrize(
+    ("secrets", "offender", "unprintable"),
+    (
+        ('{"AWS_ACCESS_KEY_ID": "AKIAEXAMPLE", "BAD-NAME": "shibboleth"}', "BAD-NAME", "shibboleth"),
+        ("""{"AWS_SECRET_ACCESS_KEY": "quote'shibboleth"}""", "AWS_SECRET_ACCESS_KEY", "shibboleth"),
+        ('{"DD_API_KEY": "line\\nshibboleth"}', "DD_API_KEY", "shibboleth"),
+    ),
+)
+def test_an_unusable_secret_is_named_without_printing_its_value(
+    tmp_path: Path, secrets: str, offender: str, unprintable: str
+) -> None:
+    env_path: Final = tmp_path / ".env"
+
+    result: Final = subprocess.run(
+        [sys.executable, str(SECRETS_TO_ENV), str(env_path)], input=secrets, capture_output=True, text=True
+    )
+
+    assert result.returncode == 1
+    assert offender in result.stderr
+    assert unprintable not in result.stderr
+    assert result.stdout == ""
+    assert not env_path.exists()

tests/e2e/CONTRIBUTING.md

@@ -54,6 +54,20 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 A couple of logging destinations are configured on the proxy rather than by the test. The Weave tests scope their callback to the key they create, but litellm builds the `weave_otel` logger from `WANDB_API_KEY` and `WANDB_PROJECT_ID` before it applies the per-key vars, so the proxy needs both in its own environment or the key-scoped callback never initializes and nothing ships
 
+### The pull request check
+
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start
+
+Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
+
+Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code
+
+Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need
+
+Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, pytest's summary line, failed test ids, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
+
+To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
+
 ### Record and replay
 
 Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop

tests/e2e/claude_code/_env.py

@@ -99,5 +99,6 @@ def require_proxy_client(
         base_url=cfg.base_url,
         master_key=cfg.api_key,
         control_plane_base_url=cfg.base_url,
+        replica_urls=(cfg.base_url,),
     )
     return ProxyClientConfig(client=client, api_key=cfg.api_key)

tests/e2e/claude_code/conftest.py

@@ -594,6 +594,7 @@ def _build_control_plane_client(proxy_config: ProxyConfig):
         base_url=proxy_config.base_url,
         master_key=proxy_config.api_key,
         control_plane_base_url=proxy_config.base_url,
+        replica_urls=(proxy_config.base_url,),
     )
 
 

tests/e2e/coverage_registry/collector.py

@@ -105,6 +105,8 @@ def collect_markers(e2e_dir: Path = E2E_DIR) -> CollectedMarkers:
                 "--continue-on-collection-errors",
                 "-p",
                 "no:cacheprovider",
+                "-p",
+                "no:pytest-retry",
                 str(e2e_dir),
             ],
             plugins=[sink],

tests/e2e/e2e_config.py

@@ -33,6 +33,14 @@ CONTROL_PLANE_BASE_URL = os.environ.get(
     "LITELLM_CONTROL_PLANE_URL", PROXY_BASE_URL
 ).rstrip("/")
 
+
+def parse_replica_urls(raw: str, fallback: str) -> tuple[str, ...]:
+    urls: Final = tuple(url.strip().rstrip("/") for url in raw.split(",") if url.strip())
+    return urls or (fallback,)
+
+
+PROXY_REPLICA_URLS: Final = parse_replica_urls(os.environ.get("LITELLM_PROXY_REPLICA_URLS", ""), PROXY_BASE_URL)
+
 UI_USERNAME = os.environ.get("E2E_UI_USERNAME", "admin")
 UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY)
 
@@ -87,10 +95,11 @@ SLOW_PROVIDER_TIMEOUT_SECONDS = float(os.environ.get("E2E_SLOW_PROVIDER_TIMEOUT"
 # (`proxy_config_reload_interval_seconds`, 30s by default and 7s on the e2e stack)
 # plus margin.
 #
-# The barriers below wait this out instead of returning on first sight, because a
-# single successful read only proves ONE replica converged: every request opens a
-# fresh connection, so a load-balanced Service routes each one independently and
-# the next call re-rolls. See ProxyClient._await_model_servable.
+# The barriers below wait this out on top of polling /v1/models on every replica in
+# PROXY_REPLICA_URLS: that poll proves each addressed gateway converged, but not the
+# workers behind it, and behind a load balancer (PROXY_REPLICA_URLS unset) a
+# successful read only proves ONE replica converged, because every request opens a
+# fresh connection and the next call re-rolls. See ProxyClient._await_model_servable.
 PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15"))
 
 EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes")

tests/e2e/gateway/stage_mirror_ci_config.yml

@@ -0,0 +1,64 @@
+general_settings:
+  proxy_config_reload_interval_seconds: 7
+  store_prompts_in_spend_logs: true
+  database_connection_pool_limit: 10
+  forward_client_headers_to_llm_api: false
+  maximum_spend_logs_retention_period: "60d"
+  maximum_spend_logs_cleanup_cron: "0 1 * * *"
+  proxy_budget_rescheduler_min_time: 15
+  proxy_budget_rescheduler_max_time: 20
+
+litellm_settings:
+  drop_params: true
+  default_redis_ttl: 20
+  request_timeout: 600
+  num_retries: 3
+  json_logs: true
+  store_audit_logs: true
+  cache: true
+  cache_params:
+    type: redis
+    host: 127.0.0.1
+    port: 6379
+    redis_startup_nodes:
+      - host: 127.0.0.1
+        port: 6379
+    ssl: true
+  callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"]
+  require_auth_for_metrics_endpoint: false
+
+router_settings:
+  routing_strategy: simple-shuffle
+  num_retries: 3
+  allowed_fails: 5
+  cooldown_time: 30
+
+model_list:
+  - model_name: gpt-5.5
+    litellm_params:
+      model: openai/gpt-5.5
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: claude-haiku-4-5
+    litellm_params:
+      model: anthropic/claude-haiku-4-5
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: gemini-2.5-flash-vertex
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: us-central1
+      vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
+  - model_name: gemini-2.5-flash
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: openai-text-embedding-3-small
+    litellm_params:
+      model: openai/text-embedding-3-small
+      api_key: os.environ/OPENAI_API_KEY
+
+mcp_servers:
+  devin:
+    url: "https://mcp.devin.ai/mcp"
+    auth_type: api_key
+    auth_value: os.environ/DEVIN_API_KEY

tests/e2e/proxy_client.py

@@ -10,12 +10,15 @@ from __future__ import annotations
 
 import time
 import warnings
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime
+from types import MappingProxyType
+from typing import Final
 
 from e2e_http import (
     AnthropicHeaders,
+    AuthHeaders,
     NoBody,
     ProbeResult,
     Result,
@@ -72,6 +75,7 @@ from e2e_config import (
     POLL_INTERVAL,
     POLL_TIMEOUT,
     PROXY_BASE_URL,
+    PROXY_REPLICA_URLS,
     REQUEST_TIMEOUT,
     SLOW_PROVIDER_TIMEOUT_SECONDS,
     settle_propagation,
@@ -80,10 +84,11 @@ from transport import HttpTransport, SplitTransport, Transport
 
 RowsPredicate = Callable[[list[SpendLogRow]], bool]
 
-# After /model/new, poll data-plane /v1/models until the model is listed (or fail).
-# Bound by MODEL_SERVABLE_TIMEOUT so a stuck reload does not burn the spend
-# poll_timeout (120s). Return on first listing; settle_propagation owns the separate
-# wait that lets every worker and replica reload before the caller uses the model.
+# After /model/new, poll /v1/models on every replica in PROXY_REPLICA_URLS until each
+# lists the model (or fail). Bound by MODEL_SERVABLE_TIMEOUT per replica so a stuck
+# reload does not burn the spend poll_timeout (120s). Return on first listing;
+# settle_propagation owns the separate wait that lets the workers behind each replica
+# reload before the caller uses the model.
 MODEL_SERVABLE_TIMEOUT = 40.0
 MODEL_SERVABLE_DB_SYNC_SECONDS = 0.0
 MODEL_SERVABLE_INTERVAL = 2.0
@@ -109,6 +114,16 @@ class NotServable:
 
 ServableOutcome = Servable | NotServable
 
+type ModelsPoller = Callable[[float], Result[ModelsListResponse]]
+
+
+@dataclass(frozen=True, slots=True)
+class NotServableOn:
+    """`NotServable` labeled with the replica whose /v1/models never listed the model."""
+
+    replica: str
+    last_result: Result[ModelsListResponse] | None
+
 
 def await_servable(
     list_models: Callable[[float], Result[ModelsListResponse]],
@@ -172,9 +187,41 @@ def await_servable(
             sleep(wait)
 
 
+def await_servable_everywhere(
+    pollers: Mapping[str, ModelsPoller],
+    *,
+    model_name: str,
+    timeout: float,
+    interval: float,
+    request_timeout: float,
+    db_sync_seconds: float,
+    now: Callable[[], float],
+    sleep: Callable[[float], None],
+) -> Servable | NotServableOn:
+    """`await_servable` against every replica in turn, each with the full budget, so
+    the model is only servable once every replica has listed it."""
+    for replica, list_models in pollers.items():
+        match await_servable(
+            list_models,
+            model_name=model_name,
+            timeout=timeout,
+            interval=interval,
+            request_timeout=request_timeout,
+            db_sync_seconds=db_sync_seconds,
+            now=now,
+            sleep=sleep,
+        ):
+            case NotServable(last_result=last_result):
+                return NotServableOn(replica=replica, last_result=last_result)
+            case Servable():
+                continue
+    return Servable()
+
+
 def servable_timeout_message(
     *,
     model_name: str,
+    replica: str,
     timeout: float,
     db_sync_seconds: float,
     last_result: Result[ModelsListResponse] | None,
@@ -185,8 +232,8 @@ def servable_timeout_message(
         else ""
     )
     return (
-        f"model {model_name!r} was created but never became servable on the data "
-        f"plane within {timeout}s of first listing (plus {db_sync_seconds}s continuous "
+        f"model {model_name!r} was created but never became servable on {replica} "
+        f"within {timeout}s of first listing (plus {db_sync_seconds}s continuous "
         f"DB sync) after /model/new (control/data-plane propagation or "
         f"STORE_MODEL_IN_DB reload issue){last_error}"
     )
@@ -195,6 +242,7 @@ def servable_timeout_message(
 @dataclass(frozen=True, slots=True)
 class ProxyClient:
     transport: Transport
+    replicas: Mapping[str, Transport]
     poll_timeout: float = 120.0
     poll_interval: float = 5.0
     model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT
@@ -311,12 +359,13 @@ class ProxyClient:
         model name passed". We poll the data-plane /v1/models until the model
         appears, then settle for the remainder of the propagation budget.
 
-        Both steps are needed, and the second is the one that matters at >1 replica.
-        The poll proves *a* replica is serving the model; it cannot prove they all
-        are, because every request opens a fresh connection and a load-balanced
-        Service routes each one independently -- so the caller's next request
-        re-rolls and can land on a replica that has not reloaded yet. Waiting out
-        PROPAGATION_TIMEOUT is what makes the model safe to use anywhere."""
+        Both steps are needed. The poll asks every replica in PROXY_REPLICA_URLS
+        directly, so behind the stack's load balancer it proves each gateway serves
+        the model rather than whichever one the balancer routed the poll to. It still
+        cannot see the workers behind a gateway, nor any replica when only the
+        balancer address is configured (every request opens a fresh connection, so
+        the caller's next request re-rolls), so waiting out PROPAGATION_TIMEOUT is
+        what makes the model safe to use anywhere."""
         model_id = unwrap(
             self.transport.post(
                 "/model/new",
@@ -331,16 +380,10 @@ class ProxyClient:
         return model_id
 
     def _await_model_servable(self, model_name: str, listed_for: str | None = None) -> None:
-        """Block until the data plane lists `model_name`, or fail at model_servable_timeout."""
-        headers = self.transport.master if listed_for is None else self.transport.bearer(listed_for)
-        outcome = await_servable(
-            lambda poll_timeout: self.transport.get(
-                "/v1/models",
-                headers=headers,
-                params=ModelsListParams(),
-                response_type=ModelsListResponse,
-                timeout=poll_timeout,
-            ),
+        """Block until every replica lists `model_name`, or fail at model_servable_timeout."""
+        headers: Final = self.transport.master if listed_for is None else self.transport.bearer(listed_for)
+        outcome: Final = await_servable_everywhere(
+            {url: self._models_poller(transport, headers) for url, transport in self.replicas.items()},
             model_name=model_name,
             timeout=self.model_servable_timeout,
             interval=self.model_servable_interval,
@@ -352,16 +395,27 @@ class ProxyClient:
         match outcome:
             case Servable():
                 return
-            case NotServable(last_result=last_result):
+            case NotServableOn(replica=replica, last_result=last_result):
                 raise AssertionError(
                     servable_timeout_message(
                         model_name=model_name,
+                        replica=replica,
                         timeout=self.model_servable_timeout,
                         db_sync_seconds=self.model_servable_db_sync_seconds,
                         last_result=last_result,
                     )
                 )
 
+    @staticmethod
+    def _models_poller(transport: Transport, headers: AuthHeaders) -> ModelsPoller:
+        return lambda poll_timeout: transport.get(
+            "/v1/models",
+            headers=headers,
+            params=ModelsListParams(),
+            response_type=ModelsListResponse,
+            timeout=poll_timeout,
+        )
+
     def update_model(self, model_id: str, litellm_params: LiteLLMParamsBody) -> None:
         """Merge `litellm_params` over the deployment `model_id`'s stored params via
         POST /model/update. The proxy overlays only the non-null fields and clears
@@ -548,16 +602,20 @@ def build_proxy_client(
     base_url: str = PROXY_BASE_URL,
     master_key: str = MASTER_KEY,
     control_plane_base_url: str = CONTROL_PLANE_BASE_URL,
+    replica_urls: tuple[str, ...] = PROXY_REPLICA_URLS,
 ) -> ProxyClient:
     """The ProxyClient every suite's client is built from: a SplitTransport that routes
     LLM calls to the data plane (PROXY_BASE_URL) and management/admin calls to the
     control plane (CONTROL_PLANE_BASE_URL), with the shared poll budget. The two
     base URLs are the same for a monolithic proxy, so routing is then a no-op.
+    ``replica_urls`` (PROXY_REPLICA_URLS) names every data-plane replica the model
+    barrier polls directly; it is the data-plane URL itself unless the stack
+    exports each gateway's own address.
 
     The endpoints are injectable for callers that resolve the proxy some other
     way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must
-    pass all three together, since a caller that overrides only the data plane
-    would leave management calls pointed at the env default.
+    pass all four together, since a caller that overrides only the data plane
+    would leave management calls and the replica poll pointed at the env defaults.
 
     Test-to-proxy traffic always goes over the wire, in every E2E_FIXTURE_MODE:
     record and replay scope to the proxy's provider-bound calls via the
@@ -574,8 +632,15 @@ def build_proxy_client(
             request_timeout=REQUEST_TIMEOUT,
         ),
     )
+    replicas: Final = MappingProxyType(
+        {
+            url: HttpTransport(base_url=url, master_key=master_key, request_timeout=REQUEST_TIMEOUT)
+            for url in replica_urls
+        }
+    )
     return ProxyClient(
         transport=split,
+        replicas=replicas,
         poll_timeout=POLL_TIMEOUT,
         poll_interval=POLL_INTERVAL,
     )

tests/e2e/test_proxy_client.py

@@ -0,0 +1,87 @@
+"""Harness coverage for the model barrier that gates on every replica.
+
+No proxy needed and no ``e2e`` marker: this pins that a model registered through
+the control plane only counts as servable once every configured replica lists it
+on /v1/models, which is what keeps a two-gateway stack from handing a test a
+model that one gateway has not reloaded yet. The fakes are plain pollers and an
+injected clock, so nothing here monkeypatches anything.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from itertools import chain, repeat
+from typing import Final
+
+import pytest
+
+from e2e_config import parse_replica_urls
+from e2e_http import Success
+from models import ModelListEntry, ModelsListResponse
+from proxy_client import ModelsPoller, NotServableOn, Servable, await_servable_everywhere
+
+MODEL: Final = "gpt-under-test"
+TIMEOUT: Final = 10.0
+INTERVAL: Final = 2.0
+
+
+@dataclass
+class FakeClock:
+    elapsed: float = 0.0
+
+    def now(self) -> float:
+        return self.elapsed
+
+    def sleep(self, seconds: float) -> None:
+        self.elapsed += seconds
+
+
+def _listing(*model_ids: str) -> Success[ModelsListResponse]:
+    entries: Final = tuple(ModelListEntry(id=model_id) for model_id in model_ids)
+    return Success(status_code=200, data=ModelsListResponse(data=entries))
+
+
+def _poller(results: Iterable[Success[ModelsListResponse]]) -> ModelsPoller:
+    it: Final = iter(results)
+    return lambda _timeout: next(it)
+
+
+def _await(pollers: Mapping[str, ModelsPoller]) -> Servable | NotServableOn:
+    clock: Final = FakeClock()
+    return await_servable_everywhere(
+        pollers,
+        model_name=MODEL,
+        timeout=TIMEOUT,
+        interval=INTERVAL,
+        request_timeout=5.0,
+        db_sync_seconds=0.0,
+        now=clock.now,
+        sleep=clock.sleep,
+    )
+
+
+class TestAwaitServableEverywhere:
+    @pytest.mark.parametrize("missing", ["gateway-1", "gateway-2"])
+    def test_fails_on_the_replica_that_never_lists_the_model(self, missing: str) -> None:
+        pollers: Final = {
+            "gateway-1": _poller(repeat(_listing(MODEL))),
+            "gateway-2": _poller(repeat(_listing(MODEL))),
+        } | {missing: _poller(repeat(_listing()))}
+        assert _await(pollers) == NotServableOn(replica=missing, last_result=_listing())
+
+    def test_passes_once_every_replica_lists_the_model(self) -> None:
+        pollers: Final = {
+            "gateway-1": _poller(repeat(_listing(MODEL))),
+            "gateway-2": _poller(chain(repeat(_listing(), 2), repeat(_listing(MODEL)))),
+        }
+        assert _await(pollers) == Servable()
+
+
+class TestParseReplicaUrls:
+    def test_splits_and_trims_the_gateway_addresses(self) -> None:
+        raw: Final = " http://127.0.0.1:4010/, http://127.0.0.1:4011 "
+        assert parse_replica_urls(raw, "http://lb") == ("http://127.0.0.1:4010", "http://127.0.0.1:4011")
+
+    def test_falls_back_to_the_data_plane_address_when_unset(self) -> None:
+        assert parse_replica_urls("", "http://lb") == ("http://lb",)

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py

@@ -40,6 +40,51 @@ from litellm.types.utils import (
 )
 
 
+def test_translate_chat_refusal_to_anthropic_response():
+    response = ModelResponse(
+        id="chatcmpl-refusal",
+        model="openai-model",
+        choices=[
+            Choices(
+                index=0,
+                finish_reason="stop",
+                message=Message(content=None, role="assistant", refusal="I cannot fulfill this request."),
+            )
+        ],
+        usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
+    )
+
+    result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
+
+    assert result["content"] == [{"type": "text", "text": "I cannot fulfill this request."}]
+    assert result["stop_reason"] == "refusal"
+    assert result.get("stop_details") == {
+        "type": "refusal",
+        "category": None,
+        "explanation": "I cannot fulfill this request.",
+    }
+
+
+def test_translate_chat_length_takes_precedence_over_refusal():
+    response = ModelResponse(
+        id="chatcmpl-partial-refusal",
+        model="openai-model",
+        choices=[
+            Choices(
+                index=0,
+                finish_reason="length",
+                message=Message(content=None, role="assistant", refusal="Partial refusal"),
+            )
+        ],
+        usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
+    )
+
+    result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
+
+    assert result["stop_reason"] == "max_tokens"
+    assert result.get("stop_details") is None
+
+
 def test_translate_streaming_openai_chunk_to_anthropic_content_block():
     choices = [
         StreamingChoices(

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py

@@ -108,6 +108,136 @@ def _text_deltas(events: List[dict]) -> List[str]:
     ]
 
 
+def test_streaming_chat_refusal_emits_refusal_text_and_stop_details():
+    chunks = [
+        _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")),
+        _make_chunk(Delta(content=None), finish_reason="stop"),
+    ]
+    wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model")
+
+    events = _drain_sync(wrapper)
+
+    assert _text_deltas(events) == ["I cannot fulfill this request."]
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"] == {
+        "stop_reason": "refusal",
+        "stop_details": {
+            "type": "refusal",
+            "category": None,
+            "explanation": "I cannot fulfill this request.",
+        },
+    }
+
+
+@pytest.mark.asyncio
+async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async():
+    chunks = [
+        _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")),
+        _make_chunk(Delta(content=None), finish_reason="stop"),
+    ]
+    wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model")
+
+    events = await _drain_async(wrapper)
+
+    assert _text_deltas(events) == ["I cannot fulfill this request."]
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"]["stop_reason"] == "refusal"
+    assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
+
+
+def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted():
+    """Providers that do not populate ``delta.refusal`` (Azure o-series among
+    them) hand LiteLLM the refusal as an unrecognized field, which lands in
+    ``provider_specific_fields``. That first delta still has to stream as text,
+    otherwise the client gets ``stop_reason: refusal`` over an empty content
+    array and shows the user nothing.
+    """
+    chunks = [
+        _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})),
+        _make_chunk(Delta(content=None), finish_reason="stop"),
+    ]
+    wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model")
+
+    events = _drain_sync(wrapper)
+
+    assert _text_deltas(events) == ["I cannot fulfill this request."]
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"]["stop_reason"] == "refusal"
+    assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
+
+
+@pytest.mark.asyncio
+async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted_async():
+    chunks = [
+        _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})),
+        _make_chunk(Delta(content=None), finish_reason="stop"),
+    ]
+    wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model")
+
+    events = await _drain_async(wrapper)
+
+    assert _text_deltas(events) == ["I cannot fulfill this request."]
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"]["stop_reason"] == "refusal"
+    assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
+
+
+def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved():
+    """Fake-streamed responses arrive as one chunk carrying both the delta and the
+    finish_reason. The refusal has to be split off and streamed as text, or the
+    client gets ``stop_reason: refusal`` over an empty content array.
+    """
+    chunks = [
+        _make_chunk(
+            Delta(content=None, refusal="I cannot fulfill this request."),
+            finish_reason="stop",
+        )
+    ]
+    wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model")
+
+    events = _drain_sync(wrapper)
+
+    assert _text_deltas(events) == ["I cannot fulfill this request."]
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"]["stop_reason"] == "refusal"
+    assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
+
+
+@pytest.mark.asyncio
+async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async():
+    chunks = [
+        _make_chunk(
+            Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."}),
+            finish_reason="stop",
+        )
+    ]
+    wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model")
+
+    events = await _drain_async(wrapper)
+
+    assert _text_deltas(events) == ["I cannot fulfill this request."]
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"]["stop_reason"] == "refusal"
+    assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
+
+
+@pytest.mark.parametrize("async_mode", [False, True])
+@pytest.mark.asyncio
+async def test_streaming_chat_length_takes_precedence_over_refusal(async_mode: bool):
+    chunks = [
+        _make_chunk(Delta(content=None, refusal="Partial refusal")),
+        _make_chunk(Delta(content=None), finish_reason="length"),
+    ]
+    stream = _AsyncStream(chunks) if async_mode else iter(chunks)
+    wrapper = AnthropicStreamWrapper(completion_stream=stream, model="openai-model")
+
+    events = await _drain_async(wrapper) if async_mode else _drain_sync(wrapper)
+
+    message_delta = next(event for event in events if event["type"] == "message_delta")
+    assert message_delta["delta"]["stop_reason"] == "max_tokens"
+    assert "stop_details" not in message_delta["delta"]
+
+
 def _input_json_deltas(events: List[dict]) -> List[str]:
     return [
         e["delta"]["partial_json"]

tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py

@@ -34,6 +34,14 @@ def _drain_async(events: list) -> list:
     return asyncio.run(_run())
 
 
+def _drain_sync_upstream(events: list) -> list:
+    async def _run() -> list:
+        wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter(events), model="m")
+        return [chunk async for chunk in wrapper]
+
+    return asyncio.run(_run())
+
+
 class TestMessageStartEmittedExactlyOnce:
     """The ``__anext__`` fallback emits ``message_start`` before consuming the
     stream, so ``_process_event`` must not emit a second one when
@@ -55,6 +63,15 @@ class TestMessageStartEmittedExactlyOnce:
         chunks = _drain_async([{"type": "response.created"}])
         assert chunks[0]["type"] == "message_start"
 
+    def test_sync_upstream_iterator_is_consumed(self):
+        chunks = _drain_sync_upstream(
+            [
+                {"type": "response.created"},
+                {"type": "response.output_text.delta", "item_id": "m1", "delta": "hi"},
+            ]
+        )
+        assert any(chunk.get("delta", {}).get("text") == "hi" for chunk in chunks)
+
 
 class TestProcessEventResponseCreatedGuard:
     """``_process_event`` must emit ``message_start`` exactly once even if
@@ -308,3 +325,60 @@ class TestResponseCompletedUsage:
             "cache_creation_input_tokens": 10,
             "cache_read_input_tokens": 4004,
         }
+
+
+class TestRefusalStreamEvents:
+    def test_refusal_event_sequence_emits_refusal_text_and_stop_details(self):
+        response = SimpleNamespace(
+            status="completed",
+            output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}],
+            usage=None,
+        )
+        chunks = _process_all(
+            [
+                {"type": "response.created"},
+                {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}},
+                {"type": "response.refusal.delta", "item_id": "msg_1", "delta": "I cannot fulfill this."},
+                {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}},
+                {"type": "response.completed", "response": response},
+            ]
+        )
+        assert [chunk["type"] for chunk in chunks] == [
+            "message_start",
+            "content_block_start",
+            "content_block_delta",
+            "content_block_stop",
+            "message_delta",
+            "message_stop",
+        ]
+        assert chunks[2]["delta"] == {"type": "text_delta", "text": "I cannot fulfill this."}
+        assert chunks[4]["delta"] == {
+            "stop_reason": "refusal",
+            "stop_sequence": None,
+            "stop_details": {
+                "type": "refusal",
+                "category": None,
+                "explanation": "I cannot fulfill this.",
+            },
+        }
+
+    def test_response_completed_with_refusal_sets_stop_reason_refusal(self):
+        response = SimpleNamespace(
+            status="completed",
+            output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Policy violation"}]}],
+            usage=None,
+        )
+        chunks = _process_all([{"type": "response.completed", "response": response}])
+        message_delta = next(c for c in chunks if c["type"] == "message_delta")
+        assert message_delta["delta"]["stop_reason"] == "refusal"
+
+    def test_incomplete_status_takes_precedence_over_refusal(self):
+        response = SimpleNamespace(
+            status="incomplete",
+            output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Partial refusal"}]}],
+            usage=None,
+        )
+        chunks = _process_all([{"type": "response.incomplete", "response": response}])
+        message_delta = next(c for c in chunks if c["type"] == "message_delta")
+        assert message_delta["delta"]["stop_reason"] == "max_tokens"
+        assert "stop_details" not in message_delta["delta"]

tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py

@@ -147,9 +147,7 @@ class TestOutputConfigStructuredOutput:
 
     def test_output_config_format_explicit_strict_true_is_preserved(self):
         """Nested output_config.format with explicit strict=True is preserved."""
-        req = _make_request(
-            output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}}
-        )
+        req = _make_request(output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}})
         kwargs = _ADAPTER.translate_request(req)
         assert kwargs["text"]["format"]["strict"] is True
 
@@ -1207,6 +1205,18 @@ def _make_output_message(texts: List[str]) -> MagicMock:
     return msg
 
 
+def _make_refusal_message(refusal_text: str):
+    from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
+
+    return ResponseOutputMessage(
+        id="msg_refusal",
+        content=[ResponseOutputRefusal(type="refusal", refusal=refusal_text)],
+        role="assistant",
+        status="completed",
+        type="message",
+    )
+
+
 def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock:
     """Build a mock ResponseFunctionToolCall."""
     from openai.types.responses import ResponseFunctionToolCall  # type: ignore[import]
@@ -1280,6 +1290,41 @@ class TestTranslateResponse:
         result: Any = _ADAPTER.translate_response(response)
         assert result["stop_reason"] == "end_turn"
 
+    def test_refusal_part_becomes_text_block_and_sets_stop_reason_refusal(self):
+        response = _make_mock_response(output=[_make_refusal_message("I cannot fulfill this request.")])
+        result: Any = _ADAPTER.translate_response(response)
+        assert len(result["content"]) == 1
+        assert result["content"][0]["type"] == "text"
+        assert result["content"][0]["text"] == "I cannot fulfill this request."
+        assert result["stop_reason"] == "refusal"
+        assert result.get("stop_details") == {
+            "type": "refusal",
+            "category": None,
+            "explanation": "I cannot fulfill this request.",
+        }
+
+    def test_dict_refusal_part_in_message_becomes_text_block(self):
+        output_item = {
+            "type": "message",
+            "content": [{"type": "refusal", "refusal": "Refused by policy"}],
+        }
+        response = _make_mock_response(output=[output_item])
+        result: Any = _ADAPTER.translate_response(response)
+        assert len(result["content"]) == 1
+        assert result["content"][0]["type"] == "text"
+        assert result["content"][0]["text"] == "Refused by policy"
+        assert result["stop_reason"] == "refusal"
+        assert result.get("stop_details", {}).get("explanation") == "Refused by policy"
+
+    def test_incomplete_status_takes_precedence_over_refusal(self):
+        response = _make_mock_response(
+            output=[_make_refusal_message("Partial refusal")],
+            status="incomplete",
+        )
+        result: Any = _ADAPTER.translate_response(response)
+        assert result["stop_reason"] == "max_tokens"
+        assert result.get("stop_details") is None
+
     def test_incomplete_status_sets_max_tokens(self):
         """status='incomplete' overrides stop_reason to 'max_tokens'."""
         response = _make_mock_response(
@@ -1338,9 +1383,7 @@ class TestTranslateResponse:
             ]
         )
         result: Any = _ADAPTER.translate_response(response)
-        assert result["content"] == [
-            {"type": "thinking", "thinking": "Weighing the options.", "signature": None}
-        ]
+        assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}]
 
     def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self):
         """Replaying this turn to an Anthropic model must not send a signature it cannot verify."""
@@ -1483,9 +1526,7 @@ class TestToolResultImages:
             },
             {
                 "role": "user",
-                "content": [
-                    {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}
-                ],
+                "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}],
             },
         ]
 
@@ -1632,9 +1673,7 @@ class TestToolResultDocuments:
             },
             {
                 "role": "user",
-                "content": [
-                    {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}
-                ],
+                "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}],
             },
         ]
 
@@ -1667,9 +1706,7 @@ class TestToolResultDocuments:
 
     def test_document_title_becomes_filename(self):
         output = self._tool_output(self._translate([self._base64_document(title="quarterly-report.pdf")]))
-        assert output == [
-            {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}
-        ]
+        assert output == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}]
 
     def test_url_document_becomes_file_url_part(self):
         output = self._tool_output(
@@ -1778,9 +1815,7 @@ class TestUserContentDocuments:
 
     def test_document_title_becomes_filename(self):
         content = self._user_content(self._translate([self._base64_document(title="quarterly-report.pdf")]))
-        assert content == [
-            {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}
-        ]
+        assert content == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}]
 
     def test_url_document_becomes_file_url_part(self):
         content = self._user_content(
@@ -1810,9 +1845,7 @@ class TestUserContentDocuments:
         assert content == [{"type": "input_text", "text": "still here"}]
 
     def test_document_breakpoint_rides_on_the_file_part(self):
-        content = self._user_content(
-            self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)])
-        )
+        content = self._user_content(self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)]))
         assert content == [
             {
                 "type": "input_file",
@@ -1859,7 +1892,9 @@ class TestPromptCacheBreakpointToResponses:
         ]
 
     def test_system_without_breakpoint_still_becomes_instructions(self):
-        request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}])
+        request = _make_request(
+            system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}]
+        )
         kwargs = _ADAPTER.translate_request(request)
         assert kwargs["instructions"] == "Be concise.\nBe helpful."
         assert kwargs["input"] == [

tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py

@@ -1738,3 +1738,44 @@ def test_vertex_text_embedding_request_includes_labels_from_metadata():
         },
     )
     assert req.get("labels") == {"project_id": "cost-center-1"}
+
+
+@pytest.mark.parametrize(
+    ("model", "expected_api"),
+    [
+        ("lyria-002", "lyria_predict"),
+        ("vertex_ai/lyria-002", "lyria_predict"),
+        ("lyria-3-clip-preview", "lyria_interactions"),
+        ("lyria-3-pro-preview", "lyria_interactions"),
+    ],
+)
+def test_get_vertex_ai_lyria_model_info_resolves_audio_api(model, expected_api):
+    from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info
+
+    model_info = get_vertex_ai_lyria_model_info(model=model)
+
+    assert model_info is not None
+    assert model_info["vertex_ai_audio_api"] == expected_api
+
+
+@pytest.mark.parametrize("model", ["en-US-Studio-O", "gemini-2.5-flash-preview-tts", "chirp-3-hd-charon"])
+def test_get_vertex_ai_lyria_model_info_is_none_for_non_lyria_speech_models(model):
+    from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info
+
+    assert get_vertex_ai_lyria_model_info(model=model) is None
+
+
+def test_get_vertex_ai_lyria_model_info_falls_back_to_bundled_map(monkeypatch):
+    import litellm
+    from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info
+
+    stale_runtime_model_cost = {
+        key: value for key, value in litellm.model_cost.items() if not key.startswith("vertex_ai/lyria")
+    }
+    monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost)
+
+    model_info = get_vertex_ai_lyria_model_info(model="lyria-3-pro-preview")
+
+    assert model_info is not None
+    assert model_info["vertex_ai_audio_api"] == "lyria_interactions"
+    assert model_info["supported_audio_formats"] == ("mp3", "wav")

tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py

@@ -0,0 +1,230 @@
+from datetime import datetime
+from typing import Final
+from unittest.mock import MagicMock
+
+import httpx
+import pytest
+
+import litellm
+from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
+    VertexPassthroughLoggingHandler,
+)
+from litellm.types.utils import PassthroughCallTypes
+
+
+def test_lyria_predict_response_preserves_audio_response_and_logs_cost(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(
+        litellm.model_cost,
+        "vertex_ai/lyria-002",
+        {
+            "vertex_ai_audio_api": "lyria_predict",
+            "supported_audio_formats": ["wav"],
+            "output_cost_per_image": 0.06,
+        },
+    )
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    response = httpx.Response(
+        status_code=200,
+        json={
+            "predictions": [
+                {
+                    "audioContent": "clip-1",
+                    "mimeType": "audio/wav",
+                },
+                {
+                    "audioContent": "clip-2",
+                    "mimeType": "audio/wav",
+                },
+            ]
+        },
+    )
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict",
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"instances": [{"prompt": "ambient piano"}]},
+    )
+
+    assert result["result"] == {
+        "response": {
+            "predictions": [
+                {
+                    "audioContent": "clip-1",
+                    "mimeType": "audio/wav",
+                },
+                {
+                    "audioContent": "clip-2",
+                    "mimeType": "audio/wav",
+                },
+            ]
+        }
+    }
+    assert result["kwargs"]["model"] == "lyria-002"
+    assert result["kwargs"]["custom_llm_provider"] == "vertex_ai"
+    assert result["kwargs"]["response_cost"] == pytest.approx(0.12)
+    assert logging_obj.model == "lyria-002"
+    assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.12)
+
+
+def test_audio_predict_response_uses_model_map_metadata(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(
+        litellm.model_cost,
+        "vertex_ai/music-audio-preview",
+        {
+            "vertex_ai_audio_api": "lyria_predict",
+            "supported_audio_formats": ["wav"],
+            "output_cost_per_image": 0.5,
+        },
+    )
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    response = httpx.Response(
+        status_code=200,
+        json={
+            "predictions": [
+                {
+                    "audioContent": "clip",
+                    "mimeType": "audio/wav",
+                }
+            ]
+        },
+    )
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route="/v1/projects/test/locations/us-central1/publishers/google/models/music-audio-preview:predict",
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"instances": [{"prompt": "ambient piano"}]},
+    )
+
+    assert result["kwargs"]["model"] == "music-audio-preview"
+    assert result["kwargs"]["response_cost"] == pytest.approx(0.5)
+    assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.5)
+
+
+def test_audio_predict_response_supports_bytes_base64_encoded(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(
+        litellm.model_cost,
+        "vertex_ai/lyria-002",
+        {
+            "vertex_ai_audio_api": "lyria_predict",
+            "supported_audio_formats": ["wav"],
+            "output_cost_per_image": 0.06,
+        },
+    )
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    response = httpx.Response(
+        status_code=200,
+        json={"predictions": [{"bytesBase64Encoded": "clip"}]},
+    )
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict",
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"instances": [{"prompt": "ambient piano"}]},
+    )
+
+    assert result["kwargs"]["response_cost"] == pytest.approx(0.06)
+    assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06)
+
+
+@pytest.mark.parametrize("runtime_entry_is_missing", (True, False))
+def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_incomplete(
+    monkeypatch: pytest.MonkeyPatch,
+    runtime_entry_is_missing: bool,
+    local_model_cost_map: None,
+) -> None:
+    if runtime_entry_is_missing:
+        monkeypatch.delitem(litellm.model_cost, "vertex_ai/lyria-002")
+    else:
+        monkeypatch.setitem(
+            litellm.model_cost,
+            "vertex_ai/lyria-002",
+            {
+                key: value
+                for key, value in litellm.model_cost["vertex_ai/lyria-002"].items()
+                if key != "output_cost_per_image"
+            },
+        )
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    response = httpx.Response(
+        status_code=200,
+        json={
+            "predictions": [
+                {
+                    "audioContent": "clip",
+                    "mimeType": "audio/wav",
+                }
+            ]
+        },
+    )
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict",
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"instances": [{"prompt": "ambient piano"}]},
+    )
+
+    if runtime_entry_is_missing:
+        assert "vertex_ai/lyria-002" not in litellm.model_cost
+    assert result["kwargs"]["model"] == "lyria-002"
+    assert result["kwargs"]["response_cost"] == pytest.approx(0.06)
+    assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06)
+
+
+def test_image_predict_response_is_not_billed_as_audio(
+    local_model_cost_map: None,
+) -> None:
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    response = httpx.Response(
+        status_code=200,
+        json={"predictions": [{"bytesBase64Encoded": "frame", "mimeType": "image/png"}]},
+    )
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route=(
+            "/v1/projects/test/locations/us-central1/publishers/google/models/imagen-4.0-generate-001:predict"
+        ),
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"instances": [{"prompt": "a red cube"}]},
+    )
+
+    assert isinstance(result["result"], litellm.ImageResponse)
+    assert logging_obj.call_type == PassthroughCallTypes.passthrough_image_generation.value
+    assert result["kwargs"]["response_cost"] == pytest.approx(
+        litellm.model_cost["vertex_ai/imagen-4.0-generate-001"]["output_cost_per_image"]
+    )

tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py

@@ -1,14 +1,17 @@
 import base64
+from typing import Final
 from unittest.mock import MagicMock, Mock, patch
 
 import httpx
 import pytest
 
-
 import litellm
 from litellm.llms.vertex_ai.text_to_speech.transformation import (
+    VertexAILyriaTextToSpeechConfig,
     VertexAITextToSpeechConfig,
 )
+from litellm.types.utils import LlmProviders
+from litellm.utils import ProviderConfigManager
 
 
 class TestVertexAITextToSpeechConfig:
@@ -41,9 +44,7 @@ class TestVertexAITextToSpeechConfig:
 
     @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token")
     @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url")
-    def test_transform_text_to_speech_request_body(
-        self, mock_get_token, mock_ensure_token
-    ):
+    def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token):
         """Test that transform_text_to_speech_request generates correct request body"""
         # Mock authentication
         mock_ensure_token.return_value = ("mock-token", "test-project")
@@ -104,9 +105,7 @@ class TestVertexAITextToSpeechConfig:
         config = VertexAITextToSpeechConfig()
 
         # Test with a Chirp3 HD voice
-        voice_str, voice_dict = config._map_voice_to_vertex_format(
-            "en-US-Chirp3-HD-Charon"
-        )
+        voice_str, voice_dict = config._map_voice_to_vertex_format("en-US-Chirp3-HD-Charon")
 
         assert voice_str == "en-US-Chirp3-HD-Charon"
         assert voice_dict is not None
@@ -169,6 +168,391 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled():
     assert result.response.content == raw_pcm
 
 
+class TestVertexAILyriaTextToSpeechConfig:
+    @pytest.mark.parametrize(
+        "model",
+        ["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"],
+    )
+    def test_provider_config_manager_selects_lyria_config(self, model):
+        config = ProviderConfigManager.get_provider_text_to_speech_config(
+            model=model,
+            provider=LlmProviders.VERTEX_AI,
+        )
+
+        assert isinstance(config, VertexAILyriaTextToSpeechConfig)
+
+    @pytest.mark.parametrize(
+        ("model", "vertex_ai_audio_api", "supported_audio_formats", "expected_url"),
+        [
+            (
+                "future-lyria-predict",
+                "lyria_predict",
+                ["wav"],
+                "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/"
+                "us-central1/publishers/google/models/future-lyria-predict:predict",
+            ),
+            (
+                "future-music-interactions",
+                "lyria_interactions",
+                ["mp3", "wav"],
+                "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions",
+            ),
+        ],
+    )
+    def test_dispatches_from_model_metadata(
+        self,
+        monkeypatch,
+        model,
+        vertex_ai_audio_api,
+        supported_audio_formats,
+        expected_url,
+    ):
+        monkeypatch.setitem(
+            litellm.model_cost,
+            f"vertex_ai/{model}",
+            {
+                "vertex_ai_audio_api": vertex_ai_audio_api,
+                "supported_audio_formats": supported_audio_formats,
+            },
+        )
+
+        config = ProviderConfigManager.get_provider_text_to_speech_config(
+            model=model,
+            provider=LlmProviders.VERTEX_AI,
+        )
+
+        assert isinstance(config, VertexAILyriaTextToSpeechConfig)
+        assert (
+            config.get_complete_url(
+                model=model,
+                api_base=None,
+                litellm_params={
+                    "vertex_project": "music-project",
+                    "vertex_location": "us-central1",
+                },
+            )
+            == expected_url
+        )
+
+    def test_vertex_chirp_does_not_select_lyria_config(self):
+        config = ProviderConfigManager.get_provider_text_to_speech_config(
+            model="chirp",
+            provider=LlmProviders.VERTEX_AI,
+        )
+
+        assert isinstance(config, VertexAITextToSpeechConfig)
+        assert not isinstance(config, VertexAILyriaTextToSpeechConfig)
+
+    def test_get_complete_url_for_lyria_2(self):
+        config = VertexAILyriaTextToSpeechConfig()
+
+        url = config.get_complete_url(
+            model="lyria-002",
+            api_base=None,
+            litellm_params={
+                "vertex_project": "music-project",
+                "vertex_location": "europe-west4",
+            },
+        )
+
+        assert url == (
+            "https://europe-west4-aiplatform.googleapis.com/v1/projects/music-project/"
+            "locations/europe-west4/publishers/google/models/lyria-002:predict"
+        )
+
+    def test_get_complete_url_encodes_injected_predict_path_segments(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        injected: Final = (
+            "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored="
+        )
+        encoded: Final = (
+            "victim-project%2Flocations%2Fus-central1%2Fpublishers%2Fgoogle"
+            "%2Fmodels%2Fother-model%3Apredict%3Fignored%3D"
+        )
+        monkeypatch.setitem(
+            litellm.model_cost,
+            f"vertex_ai/{injected}",
+            {
+                "vertex_ai_audio_api": "lyria_predict",
+                "supported_audio_formats": ["wav"],
+            },
+        )
+
+        url: Final = VertexAILyriaTextToSpeechConfig().get_complete_url(
+            model=injected,
+            api_base="https://us-central1-aiplatform.googleapis.com",
+            litellm_params={
+                "vertex_project": injected,
+                "vertex_location": injected,
+            },
+        )
+
+        assert url == (
+            "https://us-central1-aiplatform.googleapis.com"
+            f"/v1/projects/{encoded}/locations/{encoded}/publishers/google/models/{encoded}:predict"
+        )
+
+    def test_get_complete_url_for_lyria_3(self):
+        config = VertexAILyriaTextToSpeechConfig()
+
+        url = config.get_complete_url(
+            model="lyria-3-pro-preview",
+            api_base=None,
+            litellm_params={"vertex_project": "music-project"},
+        )
+
+        assert url == ("https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions")
+
+    @pytest.mark.parametrize(
+        ("model", "response_format", "expected_body"),
+        [
+            (
+                "lyria-002",
+                "wav",
+                {
+                    "instances": [{"prompt": "A bright synth track"}],
+                    "parameters": {"sample_count": 1},
+                },
+            ),
+            (
+                "lyria-3-clip-preview",
+                "mp3",
+                {
+                    "model": "lyria-3-clip-preview",
+                    "input": "A bright synth track",
+                },
+            ),
+            (
+                "lyria-3-pro-preview",
+                "wav",
+                {
+                    "model": "lyria-3-pro-preview",
+                    "input": "A bright synth track",
+                    "response_format": {
+                        "type": "audio",
+                        "mime_type": "audio/wav",
+                    },
+                },
+            ),
+        ],
+    )
+    def test_transform_request(
+        self,
+        model,
+        response_format,
+        expected_body,
+    ):
+        class _LyriaConfig(VertexAILyriaTextToSpeechConfig):
+            def _ensure_access_token(self, *args: object, **kwargs: object) -> tuple[str, str]:
+                return "mock-token", "music-project"
+
+        config = _LyriaConfig()
+
+        request = config.transform_text_to_speech_request(
+            model=model,
+            input="A bright synth track",
+            voice="alloy",
+            optional_params={"response_format": response_format},
+            litellm_params={"vertex_project": "music-project"},
+            headers={},
+        )
+
+        assert request["dict_body"] == expected_body
+        assert request["headers"]["Authorization"] == "Bearer mock-token"
+        assert request["headers"]["x-goog-user-project"] == "music-project"
+
+    @pytest.mark.parametrize(
+        ("model", "response_json", "expected_audio", "expected_mime_type"),
+        [
+            (
+                "lyria-002",
+                {
+                    "predictions": [
+                        {
+                            "bytesBase64Encoded": "UklGRiQAAABXQVZFZm10IA==",
+                        }
+                    ]
+                },
+                b"RIFF$\x00\x00\x00WAVEfmt ",
+                "audio/wav",
+            ),
+            (
+                "lyria-3-pro-preview",
+                {
+                    "steps": [
+                        {
+                            "type": "model_output",
+                            "content": [
+                                {"type": "text", "text": "Generated lyrics"},
+                                {
+                                    "type": "audio",
+                                    "data": "bHlyaWEtMy1hdWRpbw==",
+                                    "mime_type": "audio/mpeg",
+                                },
+                            ],
+                        }
+                    ]
+                },
+                b"lyria-3-audio",
+                "audio/mpeg",
+            ),
+            (
+                "lyria-3-clip-preview",
+                {
+                    "outputs": [
+                        {"type": "text", "text": "Generated lyrics"},
+                        {
+                            "type": "audio",
+                            "data": "bHlyaWEtMy1hdWRpbw==",
+                            "mime_type": "audio/mpeg",
+                        },
+                    ]
+                },
+                b"lyria-3-audio",
+                "audio/mpeg",
+            ),
+            (
+                "lyria-3-pro-preview",
+                {
+                    "outputs": [
+                        {
+                            "type": "audio",
+                            "data": "UklGRiQAAABXQVZFZm10IA==",
+                        }
+                    ]
+                },
+                b"RIFF$\x00\x00\x00WAVEfmt ",
+                "audio/wav",
+            ),
+        ],
+    )
+    def test_transform_response(
+        self,
+        model,
+        response_json,
+        expected_audio,
+        expected_mime_type,
+    ):
+        config = VertexAILyriaTextToSpeechConfig()
+        raw_response = httpx.Response(200, json=response_json)
+
+        response = config.transform_text_to_speech_response(
+            model=model,
+            raw_response=raw_response,
+            logging_obj=MagicMock(),
+        )
+
+        assert response.content == expected_audio
+        assert response.response.headers["content-type"] == expected_mime_type
+
+    @pytest.mark.parametrize(
+        ("model", "response_format"),
+        [
+            ("lyria-002", "mp3"),
+            ("lyria-3-clip-preview", "wav"),
+            ("lyria-3-pro-preview", "opus"),
+        ],
+    )
+    def test_rejects_unsupported_response_format(self, model, response_format):
+        config = VertexAILyriaTextToSpeechConfig()
+
+        with pytest.raises(litellm.UnsupportedParamsError):
+            config.map_openai_params(
+                model=model,
+                optional_params={"response_format": response_format},
+            )
+
+    @pytest.mark.parametrize("param", ["speed", "instructions"])
+    def test_rejects_unsupported_openai_params(self, param):
+        config = VertexAILyriaTextToSpeechConfig()
+
+        with pytest.raises(litellm.UnsupportedParamsError):
+            config.map_openai_params(
+                model="lyria-3-pro-preview",
+                optional_params={param: "unsupported"},
+            )
+
+    @pytest.mark.parametrize(
+        ("model", "response_format", "response_json", "expected_url", "expected_body"),
+        [
+            (
+                "lyria-002",
+                "wav",
+                {
+                    "predictions": [
+                        {
+                            "audioContent": "bHlyaWEtMi1hdWRpbw==",
+                            "mimeType": "audio/wav",
+                        }
+                    ]
+                },
+                "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/us-central1/publishers/google/models/lyria-002:predict",
+                {
+                    "instances": [{"prompt": "A bright synth track"}],
+                    "parameters": {"sample_count": 1},
+                },
+            ),
+            (
+                "lyria-3-pro-preview",
+                "mp3",
+                {
+                    "steps": [
+                        {
+                            "type": "model_output",
+                            "content": [
+                                {
+                                    "type": "audio",
+                                    "data": "bHlyaWEtMy1hdWRpbw==",
+                                    "mime_type": "audio/mpeg",
+                                }
+                            ],
+                        }
+                    ]
+                },
+                "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions",
+                {
+                    "model": "lyria-3-pro-preview",
+                    "input": "A bright synth track",
+                },
+            ),
+        ],
+    )
+    def test_litellm_speech_dispatches_to_lyria_api(
+        self,
+        model,
+        response_format,
+        response_json,
+        expected_url,
+        expected_body,
+    ):
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.status_code = 200
+        mock_response.json.return_value = response_json
+        with (
+            patch.object(  # test-quality-ok: litellm.speech has no seam for Vertex token minting
+                VertexAILyriaTextToSpeechConfig,
+                "_ensure_access_token",
+                return_value=("mock-token", "music-project"),
+            ),
+            patch(  # test-quality-ok: litellm.speech has no seam for the HTTP handler
+                "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post",
+                return_value=mock_response,
+            ) as mock_post,
+        ):
+            response = litellm.speech(
+                model=f"vertex_ai/{model}",
+                input="A bright synth track",
+                voice="alloy",
+                response_format=response_format,
+                vertex_project="music-project",
+                vertex_location="us-central1",
+            )
+
+        assert response.content in {b"lyria-2-audio", b"lyria-3-audio"}
+        mock_post.assert_called_once()
+        assert mock_post.call_args.kwargs["url"] == expected_url
+        assert mock_post.call_args.kwargs["json"] == expected_body
+
+
 @patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post")
 @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token")
 @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url")
@@ -182,9 +566,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_
 
     # Mock HTTP response
     mock_response = Mock(spec=httpx.Response)
-    mock_response.content = (
-        b'{"audioContent": "SGVsbG8gV29ybGQ="}'  # base64 encoded "Hello World"
-    )
+    mock_response.content = b'{"audioContent": "SGVsbG8gV29ybGQ="}'  # base64 encoded "Hello World"
     mock_response.status_code = 200
     mock_response.headers = {"content-type": "application/json"}
     mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="}
@@ -203,9 +585,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_
     call_kwargs = mock_post.call_args.kwargs
 
     # Verify the URL is the Google Cloud TTS API
-    assert (
-        call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
-    )
+    assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
 
     # Verify request body structure
     assert "json" in call_kwargs

tests/test_litellm/router_strategy/test_lowest_cost.py

@@ -0,0 +1,59 @@
+from datetime import datetime
+
+import pytest
+
+import litellm
+from litellm.caching.caching import DualCache
+from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
+
+DEPLOYMENT_ID = "9876"
+KWARGS = {
+    "litellm_params": {
+        "metadata": {"model_group": "gpt-5.5-pool"},
+        "model_info": {"id": DEPLOYMENT_ID},
+    }
+}
+
+
+def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse:
+    return litellm.ModelResponse(
+        model="gpt-5.5",
+        choices=[{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}],
+        usage=litellm.Usage(prompt_tokens=12, completion_tokens=0, total_tokens=12),
+    )
+
+
+def _recorded_minute_counters(cache: DualCache) -> dict[str, int]:
+    cached = cache.get_cache(key="gpt-5.5-pool_map") or {}
+    minute_buckets = cached.get(DEPLOYMENT_ID, {})
+    assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}"
+    return next(iter(minute_buckets.values()))
+
+
+def test_log_success_event_counts_a_response_with_no_completion_tokens():
+    cache = DualCache()
+    handler = LowestCostLoggingHandler(router_cache=cache)
+
+    handler.log_success_event(
+        kwargs=KWARGS,
+        response_obj=_chat_response_with_no_completion_tokens(),
+        start_time=datetime(2026, 1, 1, 12, 0, 0),
+        end_time=datetime(2026, 1, 1, 12, 0, 2),
+    )
+
+    assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
+
+
+@pytest.mark.asyncio
+async def test_async_log_success_event_counts_a_response_with_no_completion_tokens():
+    cache = DualCache()
+    handler = LowestCostLoggingHandler(router_cache=cache)
+
+    await handler.async_log_success_event(
+        kwargs=KWARGS,
+        response_obj=_chat_response_with_no_completion_tokens(),
+        start_time=datetime(2026, 1, 1, 12, 0, 0),
+        end_time=datetime(2026, 1, 1, 12, 0, 2),
+    )
+
+    assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}

tests/test_litellm/router_strategy/test_lowest_latency.py

@@ -163,3 +163,31 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds():
     assert latencies and latencies[-1] == pytest.approx(2.0)
     assert not isinstance(latencies[-1], timedelta)
     json.dumps({"latency": latencies})
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "cached_entry",
+    [{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}],
+    ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"],
+)
+async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry):
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache)
+    cache.set_cache(
+        key="gemini-embedding-001_map",
+        value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}},
+    )
+    healthy_deployments = [
+        {"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}},
+        {"model_info": {"id": "slower"}, "litellm_params": {}},
+    ]
+
+    picked = await handler.async_get_available_deployments(
+        model_group="gemini-embedding-001",
+        healthy_deployments=healthy_deployments,
+        request_kwargs={"stream": False, "metadata": {}},
+    )
+
+    assert picked is not None
+    assert picked["model_info"]["id"] == DEPLOYMENT_ID

tests/test_litellm/test_cost_calculator.py

@@ -1,6 +1,7 @@
 
 import json
 from pathlib import Path
+from typing import Final
 
 import pytest
 
@@ -146,6 +147,51 @@ def test_cost_calculator_with_response_cost_in_additional_headers():
     assert result == 1000
 
 
+@pytest.mark.parametrize(
+    ("model", "expected_cost"),
+    [
+        ("vertex_ai/lyria-002", 0.06),
+        ("vertex_ai/lyria-3-clip-preview", 0.04),
+        ("vertex_ai/lyria-3-pro-preview", 0.08),
+    ],
+)
+@pytest.mark.parametrize("runtime_state", ("complete", "missing", "routing_only", "custom_zero", "custom_price"))
+@pytest.mark.parametrize("call_type", ("speech", "aspeech"))
+def test_vertex_lyria_speech_cost(
+    model: str,
+    expected_cost: float,
+    _local_model_cost_map: None,
+    monkeypatch: pytest.MonkeyPatch,
+    runtime_state: str,
+    call_type: str,
+) -> None:
+    model_info: Final = litellm.model_cost[model]
+    if runtime_state == "missing":
+        monkeypatch.delitem(litellm.model_cost, model)
+    elif runtime_state == "routing_only":
+        monkeypatch.setitem(
+            litellm.model_cost,
+            model,
+            {key: value for key, value in model_info.items() if key != "output_cost_per_image"},
+        )
+    elif runtime_state in ("custom_zero", "custom_price"):
+        multiplier: Final = 0 if runtime_state == "custom_zero" else 2
+        monkeypatch.setitem(
+            litellm.model_cost,
+            model,
+            {**model_info, "output_cost_per_image": model_info["output_cost_per_image"] * multiplier},
+        )
+
+    cost: Final = completion_cost(
+        model=model,
+        prompt="A bright synth track",
+        call_type=call_type,
+    )
+
+    expected: Final = 0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1)
+    assert cost == pytest.approx(expected)
+
+
 def test_baseten_model_api_pricing_entries(_local_model_cost_map):
 
     expected_pricing = {

tests/test_litellm/test_utils.py

@@ -1091,6 +1091,17 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "supports_sampling_params": {"type": "boolean"},
                 "supports_output_config": {"type": "boolean"},
                 "supports_speed": {"type": "boolean"},
+                "supported_audio_formats": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "enum": ["mp3", "wav"],
+                    },
+                },
+                "vertex_ai_audio_api": {
+                    "type": "string",
+                    "enum": ["lyria_predict", "lyria_interactions"],
+                },
                 "bedrock_output_config_effort_ceiling": {
                     "type": "string",
                     "enum": ["low", "medium", "high", "max", "xhigh"],
@@ -1113,6 +1124,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                             "/v1/images/variations",
                             "/v1/images/edits",
                             "/v1/batch",
+                            "/v1beta/interactions",
                             "/v1/audio/transcriptions",
                             "/v1/audio/speech",
                             "/v1/ocr",
@@ -2879,6 +2891,60 @@ def test_gemini_lyria_3_preview_models_in_cost_map():
     assert clip["output_cost_per_image"] == 0.04
 
 
+def test_vertex_ai_lyria_models_in_cost_map():
+    import json
+    from pathlib import Path
+
+    json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
+    with open(json_path) as f:
+        model_cost = json.load(f)
+
+    lyria_2 = model_cost.get("vertex_ai/lyria-002")
+    clip = model_cost.get("vertex_ai/lyria-3-clip-preview")
+    pro = model_cost.get("vertex_ai/lyria-3-pro-preview")
+
+    assert lyria_2 is not None
+    assert clip is not None
+    assert pro is not None
+    assert lyria_2["litellm_provider"] == "vertex_ai"
+    assert clip["litellm_provider"] == "vertex_ai"
+    assert pro["litellm_provider"] == "vertex_ai"
+    assert lyria_2["mode"] == "audio_speech"
+    assert clip["mode"] == "audio_speech"
+    assert pro["mode"] == "audio_speech"
+    assert lyria_2["output_cost_per_image"] == 0.06
+    assert lyria_2["supported_modalities"] == ["text"]
+    assert lyria_2["supported_output_modalities"] == ["audio"]
+    assert lyria_2["supports_audio_output"] is True
+    assert lyria_2["supported_audio_formats"] == ["wav"]
+    assert lyria_2["vertex_ai_audio_api"] == "lyria_predict"
+    assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"]
+    assert clip["output_cost_per_image"] == 0.04
+    assert pro["output_cost_per_image"] == 0.08
+    assert clip["supported_audio_formats"] == ["mp3"]
+    assert pro["supported_audio_formats"] == ["mp3", "wav"]
+    assert clip["vertex_ai_audio_api"] == "lyria_interactions"
+    assert pro["vertex_ai_audio_api"] == "lyria_interactions"
+    assert clip["supported_endpoints"] == [
+        "/v1beta/interactions",
+        "/v1/audio/speech",
+    ]
+    assert pro["supported_endpoints"] == [
+        "/v1beta/interactions",
+        "/v1/audio/speech",
+    ]
+    assert clip["supported_modalities"] == ["text"]
+    assert pro["supported_modalities"] == ["text"]
+    assert clip["supports_vision"] is False
+    assert pro["supports_vision"] is False
+    assert "supports_image_input" not in clip
+    assert "supports_image_input" not in pro
+    assert clip["supported_regions"] == ["global"]
+    assert pro["supported_regions"] == ["global"]
+    assert clip["supports_audio_output"] is True
+    assert pro["supports_audio_output"] is True
+
+
 def test_model_info_for_fireworks_short_form_models():
     """
     Test that fireworks_ai short-form model entries (fireworks_ai/<model>)